Skip to content

Benchmark scalar functions on production-sized inputs #9729

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
connortsui20 wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Benchmark scalar functions on production-sized inputs #9729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 10 additions & 21 deletions vortex-array/benches/binary_ops.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
reason = "benchmark fixtures use indices that fit in the chosen widths"
)]

use std::mem::size_of;
use std::sync::LazyLock;

use divan::Bencher;
Expand All @@ -38,6 +37,10 @@ use vortex_array::dtype::DecimalDType;
use vortex_array::scalar_fn::fns::operators::Operator;
use vortex_session::VortexSession;

mod support;

use support::fixed_width_array_len;

#[global_allocator]
static GLOBAL: MiMalloc = MiMalloc;

Expand All @@ -51,15 +54,10 @@ static SESSION: LazyLock<VortexSession> = LazyLock::new(array_session);
/// Sized to keep CodSpeed simulation under 1ms per benchmark.
const LEN: usize = 4_096;

/// Primitive cases process at least this many rows and this many value bytes per varying input.
/// This lengthens narrow integer cases while keeping the current CodSpeed simulations below 1 ms.
const MIN_PRIMITIVE_LEN: usize = 16_384;
const MIN_PRIMITIVE_INPUT_BYTES: usize = 96 * 1_024;

const I8_LEN: usize = primitive_len::<i8>();
const I16_LEN: usize = primitive_len::<i16>();
const I32_LEN: usize = primitive_len::<i32>();
const I64_LEN: usize = primitive_len::<i64>();
const I8_LEN: usize = fixed_width_array_len::<i8>();
const I16_LEN: usize = fixed_width_array_len::<i16>();
const I32_LEN: usize = fixed_width_array_len::<i32>();
const I64_LEN: usize = fixed_width_array_len::<i64>();

/// Per-row against per-row, short and long. This is the shape the operators are tuned for, so it
/// is the one every operator is measured on.
Expand Down Expand Up @@ -317,8 +315,8 @@ fn div_decimal_i128_nullable(bencher: Bencher) {
#[vortex_bench_support::cpu_features]
#[divan::bench]
fn lt_i64_nullable(bencher: Bencher) {
let lhs = primitive_nullable(0, 7, LEN).into_array();
let rhs = primitive_nullable(1_000_000, 5, LEN).into_array();
let lhs = primitive_nullable(0, 7, I64_LEN).into_array();
let rhs = primitive_nullable(1_000_000, 5, I64_LEN).into_array();

bench_bool(bencher, lhs, rhs, Operator::Lt);
}
Expand Down Expand Up @@ -361,15 +359,6 @@ fn bench_binary<T: Executable + 'static>(
});
}

const fn primitive_len<T>() -> usize {
let input_len = MIN_PRIMITIVE_INPUT_BYTES / size_of::<T>();
if input_len > MIN_PRIMITIVE_LEN {
input_len
} else {
MIN_PRIMITIVE_LEN
}
}

fn primitive_nonnull(base: i64, len: usize) -> PrimitiveArray {
PrimitiveArray::from_iter((0..len as i64).map(|i| base + i))
}
Expand Down
49 changes: 31 additions & 18 deletions vortex-array/benches/compare.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,26 @@ use vortex_array::scalar_fn::fns::operators::Operator;
use vortex_array::validity::Validity;
use vortex_buffer::Buffer;

mod support;

use support::fixed_width_array_len;

#[global_allocator]
static GLOBAL: MiMalloc = MiMalloc;

fn main() {
divan::main();
}

// Sized to keep CodSpeed simulation under 1ms per benchmark.
// Sized to keep the more expensive CodSpeed simulations under 1ms per benchmark.
const ARRAY_SIZE: usize = 8_192;

const F32_ARRAY_SIZE: usize = fixed_width_array_len::<f32>();
const F64_ARRAY_SIZE: usize = fixed_width_array_len::<f64>();
const I64_ARRAY_SIZE: usize = fixed_width_array_len::<i64>();
const U8_ARRAY_SIZE: usize = fixed_width_array_len::<u8>();
const U64_ARRAY_SIZE: usize = fixed_width_array_len::<u64>();

fn bench_compare(bencher: Bencher, lhs: ArrayRef, rhs: ArrayRef, op: Operator) {
let session = vortex_array::array_session();
let len = lhs.len();
Expand Down Expand Up @@ -85,9 +95,9 @@ fn bool_array_nullable(rng: &mut StdRng) -> ArrayRef {
.into_array()
}

fn int_array(rng: &mut StdRng) -> ArrayRef {
fn int_array(rng: &mut StdRng, len: usize) -> ArrayRef {
let range = Uniform::new(0i64, 100_000_000).unwrap();
(0..ARRAY_SIZE)
(0..len)
.map(|_| rng.sample(range))
.collect::<Buffer<_>>()
.into_array()
Expand All @@ -96,37 +106,37 @@ fn int_array(rng: &mut StdRng) -> ArrayRef {
fn int_array_nullable(rng: &mut StdRng) -> ArrayRef {
let range = Uniform::new(0i64, 100_000_000).unwrap();
PrimitiveArray::new(
(0..ARRAY_SIZE)
(0..I64_ARRAY_SIZE)
.map(|_| rng.sample(range))
.collect::<Buffer<_>>(),
Validity::from_iter((0..ARRAY_SIZE).map(|_| rng.random_bool(0.9))),
Validity::from_iter((0..I64_ARRAY_SIZE).map(|_| rng.random_bool(0.9))),
)
.into_array()
}

fn float_array(rng: &mut StdRng) -> ArrayRef {
(0..ARRAY_SIZE)
(0..F64_ARRAY_SIZE)
.map(|_| rng.random_range(0.0f64..1.0))
.collect::<Buffer<_>>()
.into_array()
}

fn u8_array(rng: &mut StdRng) -> ArrayRef {
(0..ARRAY_SIZE)
(0..U8_ARRAY_SIZE)
.map(|_| rng.random::<u8>())
.collect::<Buffer<_>>()
.into_array()
}

fn u64_array(rng: &mut StdRng) -> ArrayRef {
(0..ARRAY_SIZE)
(0..U64_ARRAY_SIZE)
.map(|_| rng.random::<u64>())
.collect::<Buffer<_>>()
.into_array()
}

fn f32_array(rng: &mut StdRng) -> ArrayRef {
(0..ARRAY_SIZE)
(0..F32_ARRAY_SIZE)
.map(|_| rng.random_range(0.0f32..1.0))
.collect::<Buffer<_>>()
.into_array()
Expand Down Expand Up @@ -178,8 +188,8 @@ fn compare_bool_constant(bencher: Bencher) {
#[divan::bench]
fn compare_int(bencher: Bencher) {
let mut rng = StdRng::seed_from_u64(0);
let arr1 = int_array(&mut rng);
let arr2 = int_array(&mut rng);
let arr1 = int_array(&mut rng, I64_ARRAY_SIZE);
let arr2 = int_array(&mut rng, I64_ARRAY_SIZE);
bench_compare(bencher, arr1, arr2, Operator::Gte);
}

Expand All @@ -196,8 +206,8 @@ fn compare_int_nullable(bencher: Bencher) {
#[divan::bench]
fn compare_int_constant(bencher: Bencher) {
let mut rng = StdRng::seed_from_u64(0);
let arr = int_array(&mut rng);
let constant = ConstantArray::new(50_000_000i64, ARRAY_SIZE).into_array();
let arr = int_array(&mut rng, I64_ARRAY_SIZE);
let constant = ConstantArray::new(50_000_000i64, I64_ARRAY_SIZE).into_array();
bench_compare(bencher, arr, constant, Operator::Gte);
}

Expand Down Expand Up @@ -232,8 +242,8 @@ fn compare_f32(bencher: Bencher) {
#[divan::bench]
fn compare_int_eq(bencher: Bencher) {
let mut rng = StdRng::seed_from_u64(0);
let arr1 = int_array(&mut rng);
let arr2 = int_array(&mut rng);
let arr1 = int_array(&mut rng, I64_ARRAY_SIZE);
let arr2 = int_array(&mut rng, I64_ARRAY_SIZE);
bench_compare(bencher, arr1, arr2, Operator::Eq);
}

Expand Down Expand Up @@ -279,9 +289,12 @@ fn compare_string_constant(bencher: Bencher) {
}

fn struct_array(rng: &mut StdRng) -> ArrayRef {
StructArray::from_fields(&[("a", int_array(rng)), ("b", int_array(rng))])
.unwrap()
.into_array()
StructArray::from_fields(&[
("a", int_array(rng, ARRAY_SIZE)),
("b", int_array(rng, ARRAY_SIZE)),
])
.unwrap()
.into_array()
}

#[divan::bench]
Expand Down
25 changes: 25 additions & 0 deletions vortex-array/benches/support/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Shared sizing policy for array benchmarks.

use std::mem::size_of;

const DEFAULT_DATA_BLOCK_BYTES: usize = 1 << 20;
const DEFAULT_SCAN_SPLIT_ROWS: usize = 100_000;

/// Returns a production-sized row count for one fixed-width input array.
///
/// The sizing policy mirrors Vortex's default [1 MiB writer block] and [100,000-row scan split].
/// The smaller limit models the amount of one column processed by a scan task.
///
/// [1 MiB writer block]: https://github.com/vortex-data/vortex/blob/aaed723dffe1d2c54fcc7f1bbcf760726d4e8056/vortex-file/src/strategy.rs#L67-L75
/// [100,000-row scan split]: https://github.com/vortex-data/vortex/blob/aaed723dffe1d2c54fcc7f1bbcf760726d4e8056/vortex-layout/src/scan/mod.rs#L16-L19
pub const fn fixed_width_array_len<T>() -> usize {
let data_block_rows = DEFAULT_DATA_BLOCK_BYTES / size_of::<T>();
if data_block_rows < DEFAULT_SCAN_SPLIT_ROWS {
data_block_rows
} else {
DEFAULT_SCAN_SPLIT_ROWS
}
}
Loading