From b6752b49788862207fd3f74c14cecb08da89fef8 Mon Sep 17 00:00:00 2001 From: Tony Arcieri Date: Thu, 22 Oct 2026 20:06:34 -0600 Subject: [PATCH] ctutils: add integer `primitives` module Adds a set of functions extracted from RustCrypto/crypto-bigint#1360 which provide a set of reusable integer primitive operations where we were previously generating non-constant-time code on certain platforms. Validates the generated assembly is constant time using `cargo checkct`/BINSEC. --- ctutils/checkct/Cargo.lock | 2 +- ctutils/checkct/driver/src/driver.rs | 2 + .../checkct/driver/src/driver/primitives.rs | 95 ++++++++ ctutils/src/lib.rs | 2 + ctutils/src/primitives.rs | 226 ++++++++++++++++++ ctutils/tests/primitives.rs | 30 +++ 6 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 ctutils/checkct/driver/src/driver/primitives.rs create mode 100644 ctutils/src/primitives.rs create mode 100644 ctutils/tests/primitives.rs diff --git a/ctutils/checkct/Cargo.lock b/ctutils/checkct/Cargo.lock index fc3ab105..36956081 100644 --- a/ctutils/checkct/Cargo.lock +++ b/ctutils/checkct/Cargo.lock @@ -17,7 +17,7 @@ version = "0.5.4" [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.4" dependencies = [ "cmov", ] diff --git a/ctutils/checkct/driver/src/driver.rs b/ctutils/checkct/driver/src/driver.rs index 6c800ccc..14f2a590 100644 --- a/ctutils/checkct/driver/src/driver.rs +++ b/ctutils/checkct/driver/src/driver.rs @@ -11,6 +11,8 @@ macro_rules! private_integer { }}; } +mod primitives; + macro_rules! scalar_checks { ($module:ident, $ty:ty, $from_lsb:ident, $from_lt:ident, $select:ident) => { mod $module { diff --git a/ctutils/checkct/driver/src/driver/primitives.rs b/ctutils/checkct/driver/src/driver/primitives.rs new file mode 100644 index 00000000..f017db14 --- /dev/null +++ b/ctutils/checkct/driver/src/driver/primitives.rs @@ -0,0 +1,95 @@ +//! Regression cases for the primitives used by crypto-bigint. + +use super::*; +use ctutils::primitives; + +macro_rules! zero_count_checks { + ($module:ident, $ty:ty, $leading:ident, $trailing:ident, $from_nz:ident) => { + mod $module { + use super::*; + + #[checkct] + pub fn leading_zeros() { + black_box(PublicRng.next_u32()); + black_box(primitives::$leading(private_integer!($ty))); + } + + #[checkct] + pub fn trailing_zeros() { + black_box(PublicRng.next_u32()); + black_box(primitives::$trailing(private_integer!($ty))); + } + + /// Do not put an out-of-line boundary around each limb's zero count: + /// LLVM must not introduce branches when inlining it into the scan. + #[checkct] + pub fn leading_zeros_limbs() { + black_box(PublicRng.next_u32()); + let limbs = [private_integer!($ty), private_integer!($ty)]; + let mut count = 0; + let mut mask = u32::MAX; + for limb in limbs.into_iter().rev() { + count += mask & primitives::$leading(limb); + mask &= Choice::$from_nz(limb).not().to_u32_mask(); + } + black_box(count); + } + + #[checkct] + pub fn trailing_zeros_limbs() { + black_box(PublicRng.next_u32()); + let limbs = [private_integer!($ty), private_integer!($ty)]; + let mut count = 0; + let mut mask = u32::MAX; + for limb in limbs { + count += mask & primitives::$trailing(limb); + mask &= Choice::$from_nz(limb).not().to_u32_mask(); + } + black_box(count); + } + } + }; +} + +zero_count_checks!( + u32_bits, + u32, + leading_zeros_u32, + trailing_zeros_u32, + from_u32_nz +); +zero_count_checks!( + u64_bits, + u64, + leading_zeros_u64, + trailing_zeros_u64, + from_u64_nz +); + +macro_rules! addition_checks { + ($module:ident, $ty:ty, $add:ident, $mask:ident) => { + mod $module { + use super::*; + + #[checkct] + pub fn overflowing_add() { + black_box(PublicRng.next_u32()); + let (sum, carry) = primitives::$add(private_integer!($ty), private_integer!($ty)); + black_box((sum, carry.to_u8())); + } + + /// Montgomery multiplication shifts the carry into a wider accumulator. + /// Exercise that use as well as returning the carry by itself. + #[checkct] + pub fn overflowing_add_accumulate_carry() { + black_box(PublicRng.next_u32()); + let (sum, carry) = primitives::$add(private_integer!($ty), private_integer!($ty)); + let half_bits = <$ty>::BITS >> 1; + black_box(((carry.$mask() & 1) << half_bits) | (sum >> half_bits)); + } + } + }; +} + +addition_checks!(u64_add, u64, overflowing_add_u64, to_u64_mask); +addition_checks!(u128_add, u128, overflowing_add_u128, to_u128_mask); diff --git a/ctutils/src/lib.rs b/ctutils/src/lib.rs index 4ce5dba0..62b293b8 100644 --- a/ctutils/src/lib.rs +++ b/ctutils/src/lib.rs @@ -126,6 +126,8 @@ mod choice; mod ct_option; mod traits; +pub mod primitives; + pub use choice::Choice; pub use ct_option::CtOption; pub use traits::{ diff --git a/ctutils/src/primitives.rs b/ctutils/src/primitives.rs new file mode 100644 index 00000000..8943ae50 --- /dev/null +++ b/ctutils/src/primitives.rs @@ -0,0 +1,226 @@ +//! Constant-time integer primitives for cryptographic arithmetic. +//! +//! Core integer methods can lower to branches or table lookups on targets without the corresponding +//! instructions. These helpers provide alternatives if `value` is zero-bit counting and carry propagation. +//! +//! Like [`Choice`], they use optimization barriers where needed. Their generated code should still +//! be checked for the target and compiler in use. +//! +//! Native zero counts are used when supported and advertised by the target configuration. +use crate::Choice; + +/// Count leading zero bits, returning 32 if `value` is zero. +#[inline] +#[must_use] +pub const fn leading_zeros_u32(value: u32) -> u32 { + #[cfg(any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64", + target_feature = "thumb2", + target_feature = "zbb" + ))] + { + value.leading_zeros() + } + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64", + target_feature = "thumb2", + target_feature = "zbb" + )))] + { + leading_zeros_u32_soft(value) + } +} + +/// Count trailing zero bits, returning 32 if `value` is zero. +#[inline] +#[must_use] +pub const fn trailing_zeros_u32(value: u32) -> u32 { + #[cfg(any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64", + target_feature = "thumb2", + target_feature = "zbb" + ))] + { + value.trailing_zeros() + } + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64", + target_feature = "thumb2", + target_feature = "zbb" + )))] + { + // Reversing bits uses fixed shifts and masks when RBIT is unavailable. + leading_zeros_u32_soft(value.reverse_bits()) + } +} + +/// Count leading zero bits, returning 64 if `value` is zero. +#[inline] +#[must_use] +#[allow(clippy::cast_possible_truncation)] +pub const fn leading_zeros_u64(value: u64) -> u32 { + #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] + { + value.leading_zeros() + } + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + { + let hi = (value >> 32) as u32; + let lo = value as u32; + let mask = Choice::from_u32_nz(hi).not().to_u32_mask(); + leading_zeros_u32(hi).wrapping_add(mask & leading_zeros_u32(lo)) + } +} + +/// Count trailing zero bits, returning 64 if `value` is zero. +#[inline] +#[must_use] +#[allow(clippy::cast_possible_truncation)] +pub const fn trailing_zeros_u64(value: u64) -> u32 { + #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] + { + value.trailing_zeros() + } + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + { + let hi = (value >> 32) as u32; + let lo = value as u32; + let mask = Choice::from_u32_nz(lo).not().to_u32_mask(); + trailing_zeros_u32(lo).wrapping_add(mask & trailing_zeros_u32(hi)) + } +} + +/// Fixed-stage binary search for the highest set bit, including the zero case. +#[cfg(any( + test, + not(any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "x86_64", + target_feature = "thumb2", + target_feature = "zbb" + )) +))] +#[inline] +#[allow(clippy::arithmetic_side_effects, reason = "shift is between 1 and 16")] +const fn leading_zeros_u32_soft(mut value: u32) -> u32 { + let mut count = 0u32; + let mut shift = 16; + while shift != 0 { + let mask = Choice::from_u32_nz(value >> (u32::BITS - shift)) + .not() + .to_u32_mask(); + value ^= mask & (value ^ (value << shift)); + count = count.wrapping_add(mask & shift); + shift >>= 1; + } + count.wrapping_add(Choice::from_u32_nz(value).not().to_u32_mask() & 1) +} + +/// Add two integers, returning the wrapping sum and [`Choice::TRUE`] on overflow. +#[inline] +#[must_use] +pub const fn overflowing_add_u64(lhs: u64, rhs: u64) -> (u64, Choice) { + let sum = lhs.wrapping_add(rhs); + let carry = ((lhs & rhs) | ((lhs | rhs) & !sum)) >> (u64::BITS - 1); + (sum, Choice::from_u64_lsb(carry)) +} + +/// Add two integers, returning the wrapping sum and [`Choice::TRUE`] on overflow. +#[inline] +#[must_use] +pub const fn overflowing_add_u128(lhs: u128, rhs: u128) -> (u128, Choice) { + #[cfg(target_arch = "riscv32")] + { + overflowing_add_u128_halves(lhs, rhs) + } + #[cfg(not(target_arch = "riscv32"))] + { + let sum = lhs.wrapping_add(rhs); + let carry = ((lhs & rhs) | ((lhs | rhs) & !sum)) >> (u128::BITS - 1); + (sum, Choice::from_u128_lsb(carry)) + } +} + +// RV32 can branch even in `u128::wrapping_add`, before computing the final carry. +// Keep this workaround local to that target, preserving native addition elsewhere. +#[cfg(any(target_arch = "riscv32", test))] +#[inline] +#[allow(clippy::cast_possible_truncation)] +const fn overflowing_add_u128_halves(lhs: u128, rhs: u128) -> (u128, Choice) { + let (lo, lo_carry) = overflowing_add_u64(lhs as u64, rhs as u64); + let (hi, hi_carry) = overflowing_add_u64((lhs >> 64) as u64, (rhs >> 64) as u64); + let (hi, carry) = overflowing_add_u64(hi, lo_carry.to_u8() as u64); + (((hi as u128) << 64) | lo as u128, hi_carry.or(carry)) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Keep all public entrypoints usable by const callers such as crypto-bigint. + const _: () = { + assert!(leading_zeros_u32(0) == 32); + assert!(trailing_zeros_u32(0) == 32); + assert!(leading_zeros_u64(0) == 64); + assert!(trailing_zeros_u64(0) == 64); + assert!(overflowing_add_u64(u64::MAX, 1).1.to_bool()); + assert!(overflowing_add_u128(u128::MAX, 1).1.to_bool()); + }; + + #[test] + fn zero_counts_u32() { + for bit in 0..u32::BITS { + for value in [0, 1 << bit, !(1 << bit), u32::MAX >> bit] { + assert_eq!(leading_zeros_u32(value), value.leading_zeros()); + assert_eq!(trailing_zeros_u32(value), value.trailing_zeros()); + // Exercise the software fallback even on hosts with native instructions. + assert_eq!(leading_zeros_u32_soft(value), value.leading_zeros()); + assert_eq!( + leading_zeros_u32_soft(value.reverse_bits()), + value.trailing_zeros() + ); + } + } + } + + #[test] + fn zero_counts_u64() { + for bit in 0..u64::BITS { + for value in [0, 1 << bit, !(1 << bit), u64::MAX >> bit] { + assert_eq!(leading_zeros_u64(value), value.leading_zeros()); + assert_eq!(trailing_zeros_u64(value), value.trailing_zeros()); + } + } + } + + macro_rules! addition_tests { + ($test:ident, $ty:ty, $add:ident) => { + #[test] + fn $test() { + // Include both full-width and half-width carry boundaries. + let half = 1 << (<$ty>::BITS >> 1); + let high = 1 << (<$ty>::BITS - 1); + let values = [0, 1, half - 1, half, half + 1, high - 1, high, <$ty>::MAX]; + for lhs in values { + for rhs in values { + let (sum, carry) = $add(lhs, rhs); + assert_eq!((sum, carry.to_bool()), lhs.overflowing_add(rhs)); + } + } + } + }; + } + + addition_tests!(add_u64, u64, overflowing_add_u64); + addition_tests!(add_u128, u128, overflowing_add_u128); + addition_tests!(add_u128_halves, u128, overflowing_add_u128_halves); +} diff --git a/ctutils/tests/primitives.rs b/ctutils/tests/primitives.rs new file mode 100644 index 00000000..fc053072 --- /dev/null +++ b/ctutils/tests/primitives.rs @@ -0,0 +1,30 @@ +//! Differential tests against the core integer operations. + +use ctutils::primitives::*; +use proptest::prelude::*; + +proptest! { + #[test] + fn zero_counts_u32(value in any::()) { + prop_assert_eq!(leading_zeros_u32(value), value.leading_zeros()); + prop_assert_eq!(trailing_zeros_u32(value), value.trailing_zeros()); + } + + #[test] + fn zero_counts_u64(value in any::()) { + prop_assert_eq!(leading_zeros_u64(value), value.leading_zeros()); + prop_assert_eq!(trailing_zeros_u64(value), value.trailing_zeros()); + } + + #[test] + fn add_u64(lhs in any::(), rhs in any::()) { + let (sum, carry) = overflowing_add_u64(lhs, rhs); + prop_assert_eq!((sum, carry.to_bool()), lhs.overflowing_add(rhs)); + } + + #[test] + fn add_u128(lhs in any::(), rhs in any::()) { + let (sum, carry) = overflowing_add_u128(lhs, rhs); + prop_assert_eq!((sum, carry.to_bool()), lhs.overflowing_add(rhs)); + } +}