diff --git a/Cargo.lock b/Cargo.lock index 75b167441..5d3dc25ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,6 +277,8 @@ dependencies = [ name = "sha1-checked" version = "0.11.0-rc.0" dependencies = [ + "cfg-if", + "cpufeatures", "digest", "hex-literal", "sha1", diff --git a/sha1-checked/Cargo.toml b/sha1-checked/Cargo.toml index 9dffba1b1..7835142d2 100644 --- a/sha1-checked/Cargo.toml +++ b/sha1-checked/Cargo.toml @@ -18,10 +18,14 @@ exclude = [ ] [dependencies] +cfg-if = "1.0" digest = "0.11" sha1 = { version = "0.11.0-rc.5", default-features = false } zeroize = { version = "1.8", default-features = false, optional = true } +[target.'cfg(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64"))'.dependencies] +cpufeatures = "0.3" + [dev-dependencies] digest = { version = "0.11", features = ["dev"] } hex-literal = "1" diff --git a/sha1-checked/README.md b/sha1-checked/README.md index 679c44890..a27e8a900 100644 --- a/sha1-checked/README.md +++ b/sha1-checked/README.md @@ -18,8 +18,20 @@ unsuitable for further use in any security critical capacity, as it is But, this crate provides the detection [algorithm] pioneered by git, to detect hash collisions when they occur and prevent them. The [paper] has more details on how this works. -This implementation will be slower to use than the pure SHA-1 implementation, as it has to do more computations and -it can not rely on hardware acceleration available on some CPUs. +## Performance + +This implementation is slower than plain SHA-1, since it does extra work per block to detect collisions. +Measured against this crate's own benchmarks, at throughput relative to plain, undetected SHA-1 on the same backend: + +| architecture | scalar | hardware-accelerated | +|--------------|--------|----------------------| +| `aarch64` | 64% | 63% | +| `x86_64` | 55% | 38% | + +Where the CPU's SHA-1 instructions are available, most blocks run through them, falling back to scalar +compression only when a potential collision is flagged. On `aarch64` this keeps detection roughly the same +fraction of plain SHA-1's speed as without hardware acceleration. On `x86_64`, `sha1`'s own hardware backend +speeds up by more than `sha1-checked`'s fixed per-block bookkeeping does, so the gap widens. ## Examples diff --git a/sha1-checked/src/compress.rs b/sha1-checked/src/compress.rs index 8761c74d4..bcd07ac0e 100644 --- a/sha1-checked/src/compress.rs +++ b/sha1-checked/src/compress.rs @@ -1,668 +1,9 @@ -//! Direct translation of the C code found at -//! [sha1.c](https://github.com/cr-marcstevens/sha1collisiondetection/blob/master/lib/sha1.c). -//! -//! For the original license and source details see the comments in `src/checked.rs`. +use crate::{BLOCK_SIZE, DetectionState, ubc_check::Testt}; -#![allow(clippy::many_single_char_names, clippy::too_many_arguments)] - -use crate::{ - BLOCK_SIZE, - {DetectionState, ubc_check::Testt}, -}; - -const K: [u32; 4] = [0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6]; - -#[inline(always)] -fn mix(w: &mut [u32; 80], t: usize) -> u32 { - (w[t - 3] ^ w[t - 8] ^ w[t - 14] ^ w[t - 16]).rotate_left(1) -} - -#[inline(always)] -fn f1(b: u32, c: u32, d: u32) -> u32 { - d ^ b & (c ^ d) -} - -#[inline(always)] -fn f2(b: u32, c: u32, d: u32) -> u32 { - b ^ c ^ d -} - -#[inline(always)] -fn f3(b: u32, c: u32, d: u32) -> u32 { - (b & c).wrapping_add(d & (b ^ c)) -} - -#[inline(always)] -fn f4(b: u32, c: u32, d: u32) -> u32 { - b ^ c ^ d -} - -#[inline(always)] -fn round3_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { - *e = e.wrapping_add( - a.rotate_left(5) - .wrapping_add(f3(*b, c, d)) - .wrapping_add(K[2]) - .wrapping_add(mt), - ); - *b = b.rotate_left(30); -} - -#[inline(always)] -fn round4_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { - *e = e.wrapping_add( - a.rotate_left(5) - .wrapping_add(f4(*b, c, d)) - .wrapping_add(K[3]) - .wrapping_add(mt), - ); - *b = b.rotate_left(30); -} - -#[inline(always)] -fn round1_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { - *b = b.rotate_right(30); - *e = e.wrapping_sub( - a.rotate_left(5) - .wrapping_add(f1(*b, c, d)) - .wrapping_add(K[0]) - .wrapping_add(mt), - ); -} - -#[inline(always)] -fn round2_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { - *b = b.rotate_right(30); - *e = e.wrapping_sub( - a.rotate_left(5) - .wrapping_add(f2(*b, c, d)) - .wrapping_add(K[1]) - .wrapping_add(mt), - ); -} - -#[inline(always)] -fn round3_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { - *b = b.rotate_right(30); - *e = e.wrapping_sub( - a.rotate_left(5) - .wrapping_add(f3(*b, c, d)) - .wrapping_add(K[2]) - .wrapping_add(mt), - ); -} - -#[inline(always)] -fn round4_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { - *b = b.rotate_right(30); - *e = e.wrapping_sub( - a.rotate_left(5) - .wrapping_add(f4(*b, c, d)) - .wrapping_add(K[3]) - .wrapping_add(mt), - ); -} - -#[inline(always)] -fn full_round3_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, w: &mut [u32; 80], t: usize) { - w[t] = mix(w, t); - *e = e.wrapping_add( - w[t].wrapping_add(a.rotate_left(5)) - .wrapping_add(f3(*b, c, d)) - .wrapping_add(K[2]), - ); - *b = b.rotate_left(30); -} - -#[inline(always)] -fn full_round4_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, w: &mut [u32; 80], t: usize) { - w[t] = mix(w, t); - *e = e.wrapping_add( - w[t].wrapping_add(a.rotate_left(5)) - .wrapping_add(f4(*b, c, d)) - .wrapping_add(K[3]), - ); - *b = b.rotate_left(30); -} - -#[inline] -fn round2_step4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &[u32; 80], - t: usize, -) { - // 1 - *e = e.wrapping_add( - w[t].wrapping_add(a.rotate_left(5)) - .wrapping_add(f2(*b, *c, *d)) - .wrapping_add(K[1]), - ); - *b = b.rotate_left(30); - - // 2 - *d = d.wrapping_add( - w[t + 1] - .wrapping_add(e.rotate_left(5)) - .wrapping_add(f2(*a, *b, *c)) - .wrapping_add(K[1]), - ); - *a = a.rotate_left(30); - - // 3 - *c = c.wrapping_add( - w[t + 2] - .wrapping_add(d.rotate_left(5)) - .wrapping_add(f2(*e, *a, *b)) - .wrapping_add(K[1]), - ); - *e = e.rotate_left(30); - - // 4 - *b = b.wrapping_add( - w[t + 3] - .wrapping_add(c.rotate_left(5)) - .wrapping_add(f2(*d, *e, *a)) - .wrapping_add(K[1]), - ); - *d = d.rotate_left(30); -} - -#[inline] -fn round3_step4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &[u32; 80], - t: usize, -) { - // 1 - *e = e.wrapping_add( - w[t].wrapping_add(a.rotate_left(5)) - .wrapping_add(f3(*b, *c, *d)) - .wrapping_add(K[2]), - ); - *b = b.rotate_left(30); - - // 2 - *d = d.wrapping_add( - w[t + 1] - .wrapping_add(e.rotate_left(5)) - .wrapping_add(f3(*a, *b, *c)) - .wrapping_add(K[2]), - ); - *a = a.rotate_left(30); - - // 3 - *c = c.wrapping_add( - w[t + 2] - .wrapping_add(d.rotate_left(5)) - .wrapping_add(f3(*e, *a, *b)) - .wrapping_add(K[2]), - ); - *e = e.rotate_left(30); - - // 4 - *b = b.wrapping_add( - w[t + 3] - .wrapping_add(c.rotate_left(5)) - .wrapping_add(f3(*d, *e, *a)) - .wrapping_add(K[2]), - ); - *d = d.rotate_left(30); -} - -#[inline] -fn round4_step4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &[u32; 80], - t: usize, -) { - // 1 - *e = e.wrapping_add( - w[t].wrapping_add(a.rotate_left(5)) - .wrapping_add(f4(*b, *c, *d)) - .wrapping_add(K[3]), - ); - *b = b.rotate_left(30); - - // 2 - *d = d.wrapping_add( - w[t + 1] - .wrapping_add(e.rotate_left(5)) - .wrapping_add(f4(*a, *b, *c)) - .wrapping_add(K[3]), - ); - *a = a.rotate_left(30); - - // 3 - *c = c.wrapping_add( - w[t + 2] - .wrapping_add(d.rotate_left(5)) - .wrapping_add(f4(*e, *a, *b)) - .wrapping_add(K[3]), - ); - *e = e.rotate_left(30); - - // 4 - *b = b.wrapping_add( - w[t + 3] - .wrapping_add(c.rotate_left(5)) - .wrapping_add(f4(*d, *e, *a)) - .wrapping_add(K[3]), - ); - *d = d.rotate_left(30); -} - -#[inline] -fn full_round1_step_load4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - m: &[u32; 16], - w: &mut [u32; 80], - t: usize, -) { - // load - w[t..t + 4].copy_from_slice(&m[t..t + 4]); - round1_step4(a, b, c, d, e, w, t); -} - -#[inline(always)] -fn round1_step4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &[u32; 80], - t: usize, -) { - // 1 - *e = e.wrapping_add( - w[t].wrapping_add(a.rotate_left(5)) - .wrapping_add(f1(*b, *c, *d)) - .wrapping_add(K[0]), - ); - *b = b.rotate_left(30); - - // 2 - *d = d.wrapping_add( - w[t + 1] - .wrapping_add(e.rotate_left(5)) - .wrapping_add(f1(*a, *b, *c)) - .wrapping_add(K[0]), - ); - *a = a.rotate_left(30); - - // 3 - *c = c.wrapping_add( - w[t + 2] - .wrapping_add(d.rotate_left(5)) - .wrapping_add(f1(*e, *a, *b)) - .wrapping_add(K[0]), - ); - *e = e.rotate_left(30); - - // 4 - *b = b.wrapping_add( - w[t + 3] - .wrapping_add(c.rotate_left(5)) - .wrapping_add(f1(*d, *e, *a)) - .wrapping_add(K[0]), - ); - *d = d.rotate_left(30); -} - -#[inline] -fn full_round1_step_expand4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &mut [u32; 80], - t: usize, -) { - w[t] = mix(w, t); - w[t + 1] = mix(w, t + 1); - w[t + 2] = mix(w, t + 2); - w[t + 3] = mix(w, t + 3); - round1_step4(a, b, c, d, e, w, t); -} - -#[inline] -fn full_round2_step4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &mut [u32; 80], - t: usize, -) { - w[t] = mix(w, t); - w[t + 1] = mix(w, t + 1); - w[t + 2] = mix(w, t + 2); - w[t + 3] = mix(w, t + 3); - round2_step4(a, b, c, d, e, w, t); -} - -#[inline] -fn full_round3_step4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &mut [u32; 80], - t: usize, -) { - w[t] = mix(w, t); - w[t + 1] = mix(w, t + 1); - w[t + 2] = mix(w, t + 2); - w[t + 3] = mix(w, t + 3); - round3_step4(a, b, c, d, e, w, t); -} - -#[inline] -fn full_round4_step4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - w: &mut [u32; 80], - t: usize, -) { - w[t] = mix(w, t); - w[t + 1] = mix(w, t + 1); - w[t + 2] = mix(w, t + 2); - w[t + 3] = mix(w, t + 3); - round4_step4(a, b, c, d, e, w, t); -} - -#[inline] -fn round1_step_bw4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - m: &[u32; 80], - t: usize, -) { - round1_step_bw(*a, b, *c, *d, e, m[t]); - round1_step_bw(*b, c, *d, *e, a, m[t - 1]); - round1_step_bw(*c, d, *e, *a, b, m[t - 2]); - round1_step_bw(*d, e, *a, *b, c, m[t - 3]); -} - -#[inline] -fn round2_step_bw4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - m: &[u32; 80], - t: usize, -) { - round2_step_bw(*a, b, *c, *d, e, m[t]); - round2_step_bw(*b, c, *d, *e, a, m[t - 1]); - round2_step_bw(*c, d, *e, *a, b, m[t - 2]); - round2_step_bw(*d, e, *a, *b, c, m[t - 3]); -} - -#[inline] -fn round3_step_bw4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - m: &[u32; 80], - t: usize, -) { - round3_step_bw(*a, b, *c, *d, e, m[t]); - round3_step_bw(*b, c, *d, *e, a, m[t - 1]); - round3_step_bw(*c, d, *e, *a, b, m[t - 2]); - round3_step_bw(*d, e, *a, *b, c, m[t - 3]); -} - -#[inline] -fn round4_step_bw4( - a: &mut u32, - b: &mut u32, - c: &mut u32, - d: &mut u32, - e: &mut u32, - m: &[u32; 80], - t: usize, -) { - round4_step_bw(*a, b, *c, *d, e, m[t]); - round4_step_bw(*b, c, *d, *e, a, m[t - 1]); - round4_step_bw(*c, d, *e, *a, b, m[t - 2]); - round4_step_bw(*d, e, *a, *b, c, m[t - 3]); -} - -fn add_assign(left: &mut [u32; 5], right: [u32; 5]) { - left[0] = left[0].wrapping_add(right[0]); - left[1] = left[1].wrapping_add(right[1]); - left[2] = left[2].wrapping_add(right[2]); - left[3] = left[3].wrapping_add(right[3]); - left[4] = left[4].wrapping_add(right[4]); -} - -fn compression_w(ihv: &mut [u32; 5], w: &[u32; 80]) { - let &mut [mut a, mut b, mut c, mut d, mut e] = ihv; - - round1_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 0); - round1_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 4); - round1_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 8); - round1_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 12); - round1_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 16); - - round2_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 20); - round2_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 24); - round2_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 28); - round2_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 32); - round2_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 36); - - round3_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 40); - round3_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 44); - round3_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 48); - round3_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 52); - round3_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 56); - - round4_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 60); - round4_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 64); - round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 68); - round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 72); - round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 76); - - add_assign(ihv, [a, b, c, d, e]); -} - -fn compression_states( - ihv: &mut [u32; 5], - m: &[u32; 16], - w: &mut [u32; 80], - state_58: &mut [u32; 5], - state_65: &mut [u32; 5], -) { - let &mut [mut a, mut b, mut c, mut d, mut e] = ihv; - - full_round1_step_load4(&mut a, &mut b, &mut c, &mut d, &mut e, m, w, 0); - full_round1_step_load4(&mut b, &mut c, &mut d, &mut e, &mut a, m, w, 4); - full_round1_step_load4(&mut c, &mut d, &mut e, &mut a, &mut b, m, w, 8); - full_round1_step_load4(&mut d, &mut e, &mut a, &mut b, &mut c, m, w, 12); - - full_round1_step_expand4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 16); - - full_round2_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 20); - full_round2_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 24); - full_round2_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 28); - full_round2_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 32); - full_round2_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 36); - - full_round3_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 40); - full_round3_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 44); - full_round3_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 48); - full_round3_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 52); - - full_round3_step(e, &mut a, b, c, &mut d, w, 56); - full_round3_step(d, &mut e, a, b, &mut c, w, 57); - - // Store state58 - *state_58 = [a, b, c, d, e]; - - full_round3_step(c, &mut d, e, a, &mut b, w, 58); - full_round3_step(b, &mut c, d, e, &mut a, w, 59); - - full_round4_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 60); - full_round4_step(b, &mut c, d, e, &mut a, w, 64); - - // Store state65 - *state_65 = [a, b, c, d, e]; - - full_round4_step(a, &mut b, c, d, &mut e, w, 65); - full_round4_step(e, &mut a, b, c, &mut d, w, 66); - full_round4_step(d, &mut e, a, b, &mut c, w, 67); - - full_round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 68); - full_round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 72); - full_round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 76); - - add_assign(ihv, [a, b, c, d, e]); -} - -fn recompress_fast_58( - ihvin: &mut [u32; 5], - ihvout: &mut [u32; 5], - me2: &[u32; 80], - state: &[u32; 5], -) { - let &[mut a, mut b, mut c, mut d, mut e] = state; - - round3_step_bw(d, &mut e, a, b, &mut c, me2[57]); - round3_step_bw(e, &mut a, b, c, &mut d, me2[56]); - - round3_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 55); - round3_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 51); - round3_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 47); - round3_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 43); - - round2_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 39); - round2_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 35); - round2_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 31); - round2_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 27); - round2_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 23); - - round1_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 19); - round1_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 15); - round1_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 11); - round1_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 7); - round1_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 3); - - *ihvin = [a, b, c, d, e]; - [a, b, c, d, e] = *state; - - round3_step(c, &mut d, e, a, &mut b, me2[58]); - round3_step(b, &mut c, d, e, &mut a, me2[59]); - - round4_step4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 60); - round4_step4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 64); - round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 68); - round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 72); - round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 76); - - ihvout[0] = ihvin[0].wrapping_add(a); - ihvout[1] = ihvin[1].wrapping_add(b); - ihvout[2] = ihvin[2].wrapping_add(c); - ihvout[3] = ihvin[3].wrapping_add(d); - ihvout[4] = ihvin[4].wrapping_add(e); -} - -fn recompress_fast_65( - ihvin: &mut [u32; 5], - ihvout: &mut [u32; 5], - me2: &[u32; 80], - state: &[u32; 5], -) { - let &[mut a, mut b, mut c, mut d, mut e] = state; - - round4_step_bw(b, &mut c, d, e, &mut a, me2[64]); - round4_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 63); - - round3_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 59); - round3_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 55); - round3_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 51); - round3_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 47); - round3_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 43); - - round2_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 39); - round2_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 35); - round2_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 31); - round2_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 27); - round2_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 23); - - round1_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 19); - round1_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 15); - round1_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 11); - round1_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 7); - round1_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 3); - - *ihvin = [a, b, c, d, e]; - [a, b, c, d, e] = *state; - - round4_step(a, &mut b, c, d, &mut e, me2[65]); - round4_step(e, &mut a, b, c, &mut d, me2[66]); - round4_step(d, &mut e, a, b, &mut c, me2[67]); - - round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 68); - round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 72); - round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 76); - - ihvout[0] = ihvin[0].wrapping_add(a); - ihvout[1] = ihvin[1].wrapping_add(b); - ihvout[2] = ihvin[2].wrapping_add(c); - ihvout[3] = ihvin[3].wrapping_add(d); - ihvout[4] = ihvin[4].wrapping_add(e); -} - -fn recompression_step( - step: Testt, - ihvin: &mut [u32; 5], - ihvout: &mut [u32; 5], - me2: &[u32; 80], - state: &[u32; 5], -) { - match step { - Testt::T58 => { - recompress_fast_58(ihvin, ihvout, me2, state); - } - Testt::T65 => { - recompress_fast_65(ihvin, ihvout, me2, state); - } - } -} - -#[inline(always)] -fn xor(a: &[u32; 5], b: &[u32; 5]) -> u32 { - a[0] ^ b[0] | a[1] ^ b[1] | a[2] ^ b[2] | a[3] ^ b[3] | a[4] ^ b[4] -} +mod backend; +mod soft; +use backend::Backend; +use soft::{compression_w, recompression_step, xor}; #[inline] pub(crate) fn compress( @@ -671,6 +12,7 @@ pub(crate) fn compress( blocks: &[[u8; BLOCK_SIZE]], ) { let mut block_u32 = [0u32; BLOCK_SIZE / 4]; + let backend = Backend::new(); for block in blocks.iter() { ctx.ihv1.copy_from_slice(&*state); @@ -686,7 +28,7 @@ pub(crate) fn compress( .. } = ctx; - compression_states(state, &block_u32, m1, state_58, state_65); + backend.compress(state, block, &block_u32, m1, state_58, state_65); let ubc_mask = if ctx.ubc_check { crate::ubc_check::ubc_check(&ctx.m1) @@ -695,6 +37,15 @@ pub(crate) fn compress( }; if ubc_mask != 0 { + let DetectionState { + m1, + state_58, + state_65, + ihv1, + .. + } = ctx; + backend.ensure_states(ihv1, state, &block_u32, m1, state_58, state_65); + let mut ihvtmp = [0u32; 5]; for dv_type in &crate::ubc_check::SHA1_DVS { if ubc_mask & (1 << dv_type.maskb) != 0 { diff --git a/sha1-checked/src/compress/backend.rs b/sha1-checked/src/compress/backend.rs new file mode 100644 index 000000000..0ca523537 --- /dev/null +++ b/sha1-checked/src/compress/backend.rs @@ -0,0 +1,165 @@ +//! Selects between the hardware and scalar SHA-1 compression implementations. + +use crate::BLOCK_SIZE; +use crate::compress::soft::compression_states; + +cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + mod x86_sha; + cpufeatures::new!(hwcap, "sha", "sse2", "ssse3", "sse4.1"); + } else if #[cfg(target_arch = "aarch64")] { + mod aarch64_sha2; + cpufeatures::new!(hwcap, "sha2"); + } +} + +enum Repr { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + X86Sha, + #[cfg(target_arch = "aarch64")] + Aarch64Sha2, + Scalar, +} + +/// Which implementation computes a block's digest, decided once by +/// [`Backend::new`] via cpu feature detection. +pub(super) struct Backend(Repr); + +impl Backend { + pub(super) fn new() -> Self { + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + if hwcap::get() { + return Self(Repr::X86Sha); + } + } else if #[cfg(target_arch = "aarch64")] { + if hwcap::get() { + return Self(Repr::Aarch64Sha2); + } + } + } + Self(Repr::Scalar) + } + + /// Digest `block` into `state`, writing the checker's input to `m1`. + /// + /// Depending on backend, may or may not fill in `state_58`/`state_65` too; + /// call [`ensure_states`](Self::ensure_states) before reading them. + pub(super) fn compress( + &self, + state: &mut [u32; 5], + block: &[u8; BLOCK_SIZE], + block_u32: &[u32; BLOCK_SIZE / 4], + m1: &mut [u32; 80], + state_58: &mut [u32; 5], + state_65: &mut [u32; 5], + ) { + match self.0 { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + Repr::X86Sha => { + // SAFETY: `Repr::X86Sha` implies `hwcap::get()` was checked. + unsafe { x86_sha::compress_spill(state, block, m1) }; + } + #[cfg(target_arch = "aarch64")] + Repr::Aarch64Sha2 => { + // SAFETY: `Repr::Aarch64Sha2` implies `hwcap::get()` was checked. + unsafe { aarch64_sha2::compress_spill(state, block, m1) }; + } + Repr::Scalar => { + let _ = block; // only the hardware paths need it + compression_states(state, block_u32, m1, state_58, state_65); + } + } + } + + /// Make sure `state_58`/`state_65` reflect this block, recomputing them + /// via the scalar path if needed. + pub(super) fn ensure_states( + &self, + ihv_before: &[u32; 5], + state: &[u32; 5], + block_u32: &[u32; BLOCK_SIZE / 4], + m1: &mut [u32; 80], + state_58: &mut [u32; 5], + state_65: &mut [u32; 5], + ) { + if matches!(self.0, Repr::Scalar) { + return; + } + let mut replayed = *ihv_before; + compression_states(&mut replayed, block_u32, m1, state_58, state_65); + debug_assert_eq!(replayed, *state, "hardware and scalar compression diverged"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn xorshift(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed + } + + #[test] + fn hardware_agrees_with_scalar_compression() { + let hardware = Backend::new(); + let is_hardware = match hardware.0 { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + Repr::X86Sha => true, + #[cfg(target_arch = "aarch64")] + Repr::Aarch64Sha2 => true, + Repr::Scalar => false, + }; + if !is_hardware { + return; + } + + let mut seed = 0x0BAD_C0DE_DEAD_BEEF; + for _ in 0..2_000 { + let block: [u8; BLOCK_SIZE] = + core::array::from_fn(|_| (xorshift(&mut seed) >> 24) as u8); + let ihv: [u32; 5] = core::array::from_fn(|_| xorshift(&mut seed) as u32); + + let mut block_u32 = [0u32; 16]; + for (word, chunk) in block_u32.iter_mut().zip(block.chunks_exact(4)) { + *word = u32::from_be_bytes(chunk.try_into().unwrap()); + } + + let (mut hw_state, mut hw_w) = (ihv, [0u32; 80]); + let (mut hw_s58, mut hw_s65) = ([0u32; 5], [0u32; 5]); + hardware.compress( + &mut hw_state, + &block, + &block_u32, + &mut hw_w, + &mut hw_s58, + &mut hw_s65, + ); + + let mut sc_state = ihv; + let (mut sc_w, mut s58, mut s65) = ([0u32; 80], [0u32; 5], [0u32; 5]); + Backend(Repr::Scalar).compress( + &mut sc_state, + &block, + &block_u32, + &mut sc_w, + &mut s58, + &mut s65, + ); + + assert_eq!(hw_state, sc_state, "digest diverged"); + assert_eq!(hw_w, sc_w, "schedule diverged"); + + // Independent of the scalar round macros, so a shared bug can't hide. + let mut want = [0u32; 80]; + want[..16].copy_from_slice(&block_u32); + for t in 16..80 { + want[t] = (want[t - 3] ^ want[t - 8] ^ want[t - 14] ^ want[t - 16]).rotate_left(1); + } + assert_eq!(hw_w, want, "schedule is not the standard expansion"); + } + } +} diff --git a/sha1-checked/src/compress/backend/aarch64_sha2.rs b/sha1-checked/src/compress/backend/aarch64_sha2.rs new file mode 100644 index 000000000..3e4cf4568 --- /dev/null +++ b/sha1-checked/src/compress/backend/aarch64_sha2.rs @@ -0,0 +1,189 @@ +//! SHA-1 `aarch64` backend, spilling the message schedule alongside the +//! digest. + +use crate::compress::soft::K; + +#[cfg(not(target_arch = "aarch64"))] +compile_error!("aarch64_sha2 backend can be used only on aarch64 target arches"); + +#[target_feature(enable = "sha2")] +#[allow(unsafe_op_in_unsafe_fn)] +#[allow(clippy::too_many_lines)] +pub(crate) unsafe fn compress_spill(state: &mut [u32; 5], block: &[u8; 64], w: &mut [u32; 80]) { + use core::arch::aarch64::*; + + let mut abcd = vld1q_u32(state.as_ptr()); + let mut e0 = state[4]; + let [k0, k1, k2, k3] = K.map(|k| vdupq_n_u32(k)); + let (mut e1, mut tmp0, mut tmp1); + + let abcd_cpy = abcd; + let e0_cpy = e0; + let wp = w.as_mut_ptr(); + + // Load and reverse byte order + let [mut msg0, mut msg1, mut msg2, mut msg3] = [0, 1, 2, 3].map(|i| { + let p = block.as_ptr().add(16 * i); + vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(p))) + }); + + vst1q_u32(wp, msg0); + vst1q_u32(wp.add(4), msg1); + vst1q_u32(wp.add(8), msg2); + vst1q_u32(wp.add(12), msg3); + + tmp0 = vaddq_u32(msg0, k0); + tmp1 = vaddq_u32(msg1, k0); + + // Rounds 0-3 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1cq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg2, k0); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + // Rounds 4-7 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1cq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg3, k0); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(wp.add(16), msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + // Rounds 8-11 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1cq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg0, k0); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(wp.add(20), msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + // Rounds 12-15 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1cq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg1, k1); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(wp.add(24), msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + // Rounds 16-19 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1cq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg2, k1); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(wp.add(28), msg3); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + // Rounds 20-23 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg3, k1); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(wp.add(32), msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + // Rounds 24-27 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg0, k1); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(wp.add(36), msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + // Rounds 28-31 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg1, k1); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(wp.add(40), msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + // Rounds 32-35 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg2, k2); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(wp.add(44), msg3); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + // Rounds 36-39 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg3, k2); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(wp.add(48), msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + // Rounds 40-43 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1mq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg0, k2); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(wp.add(52), msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + // Rounds 44-47 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1mq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg1, k2); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(wp.add(56), msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + // Rounds 48-51 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1mq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg2, k2); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(wp.add(60), msg3); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + // Rounds 52-55 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1mq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg3, k3); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(wp.add(64), msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + // Rounds 56-59 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1mq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg0, k3); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(wp.add(68), msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + // Rounds 60-63 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg1, k3); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(wp.add(72), msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + // Rounds 64-67 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e0, tmp0); + tmp0 = vaddq_u32(msg2, k3); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(wp.add(76), msg3); + + // Rounds 68-71 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e1, tmp1); + tmp1 = vaddq_u32(msg3, k3); + + // Rounds 72-75 + e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e0, tmp0); + + // Rounds 76-79 + e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); + abcd = vsha1pq_u32(abcd, e1, tmp1); + + abcd = vaddq_u32(abcd_cpy, abcd); + e0 = e0.wrapping_add(e0_cpy); + + vst1q_u32(state.as_mut_ptr(), abcd); + state[4] = e0; +} diff --git a/sha1-checked/src/compress/backend/x86_sha.rs b/sha1-checked/src/compress/backend/x86_sha.rs new file mode 100644 index 000000000..abcf21f3a --- /dev/null +++ b/sha1-checked/src/compress/backend/x86_sha.rs @@ -0,0 +1,100 @@ +//! SHA-1 `x86`/`x86_64` backend, spilling the message schedule alongside the +//! digest. + +#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] +compile_error!("x86_sha backend can be used only on x86 and x86_64 target arches"); + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +const REVERSE: i32 = 0b00011011; // used throughout to convert from SHA-NI lane order + +macro_rules! rounds4 { + ($h0:ident, $h1:ident, $wk:expr, $i:expr) => { + _mm_sha1rnds4_epu32($h0, _mm_sha1nexte_epu32($h1, $wk), $i) + }; +} + +macro_rules! schedule { + ($v0:expr, $v1:expr, $v2:expr, $v3:expr) => { + _mm_sha1msg2_epu32(_mm_xor_si128(_mm_sha1msg1_epu32($v0, $v1), $v2), $v3) + }; +} + +macro_rules! schedule_rounds4 { + ( + $wp:expr, $t:expr, + $h0:ident, $h1:ident, + $w0:expr, $w1:expr, $w2:expr, $w3:expr, $w4:expr, + $i:expr + ) => { + $w4 = schedule!($w0, $w1, $w2, $w3); + _mm_storeu_si128($wp.add($t).cast(), _mm_shuffle_epi32($w4, REVERSE)); + $h1 = rounds4!($h0, $h1, $w4, $i); + }; +} + +#[target_feature(enable = "sha,sse2,ssse3,sse4.1")] +#[allow(unsafe_op_in_unsafe_fn)] +pub(crate) unsafe fn compress_spill(state: &mut [u32; 5], block: &[u8; 64], w: &mut [u32; 80]) { + #[allow(non_snake_case)] + let MASK: __m128i = _mm_set_epi64x(0x0001_0203_0405_0607, 0x0809_0A0B_0C0D_0E0F); + + let wp = w.as_mut_ptr(); + let block_ptr: *const __m128i = block.as_ptr().cast(); + + let mut w0 = _mm_shuffle_epi8(_mm_loadu_si128(block_ptr.add(0)), MASK); + let mut w1 = _mm_shuffle_epi8(_mm_loadu_si128(block_ptr.add(1)), MASK); + let mut w2 = _mm_shuffle_epi8(_mm_loadu_si128(block_ptr.add(2)), MASK); + let mut w3 = _mm_shuffle_epi8(_mm_loadu_si128(block_ptr.add(3)), MASK); + #[allow(clippy::needless_late_init)] + let mut w4; + + _mm_storeu_si128(wp.add(0).cast(), _mm_shuffle_epi32(w0, REVERSE)); + _mm_storeu_si128(wp.add(4).cast(), _mm_shuffle_epi32(w1, REVERSE)); + _mm_storeu_si128(wp.add(8).cast(), _mm_shuffle_epi32(w2, REVERSE)); + _mm_storeu_si128(wp.add(12).cast(), _mm_shuffle_epi32(w3, REVERSE)); + + let state_abcd = _mm_shuffle_epi32(_mm_loadu_si128(state.as_ptr().cast()), REVERSE); + let state_e = _mm_set_epi32(state[4] as i32, 0, 0, 0); + + let mut h0 = state_abcd; + let mut h1 = _mm_add_epi32(state_e, w0); + + // Rounds 0..20 + h1 = _mm_sha1rnds4_epu32(h0, h1, 0); + h0 = rounds4!(h1, h0, w1, 0); + h1 = rounds4!(h0, h1, w2, 0); + h0 = rounds4!(h1, h0, w3, 0); + schedule_rounds4!(wp, 16, h0, h1, w0, w1, w2, w3, w4, 0); + + // Rounds 20..40 + schedule_rounds4!(wp, 20, h1, h0, w1, w2, w3, w4, w0, 1); + schedule_rounds4!(wp, 24, h0, h1, w2, w3, w4, w0, w1, 1); + schedule_rounds4!(wp, 28, h1, h0, w3, w4, w0, w1, w2, 1); + schedule_rounds4!(wp, 32, h0, h1, w4, w0, w1, w2, w3, 1); + schedule_rounds4!(wp, 36, h1, h0, w0, w1, w2, w3, w4, 1); + + // Rounds 40..60 + schedule_rounds4!(wp, 40, h0, h1, w1, w2, w3, w4, w0, 2); + schedule_rounds4!(wp, 44, h1, h0, w2, w3, w4, w0, w1, 2); + schedule_rounds4!(wp, 48, h0, h1, w3, w4, w0, w1, w2, 2); + schedule_rounds4!(wp, 52, h1, h0, w4, w0, w1, w2, w3, 2); + schedule_rounds4!(wp, 56, h0, h1, w0, w1, w2, w3, w4, 2); + + // Rounds 60..80 + schedule_rounds4!(wp, 60, h1, h0, w1, w2, w3, w4, w0, 3); + schedule_rounds4!(wp, 64, h0, h1, w2, w3, w4, w0, w1, 3); + schedule_rounds4!(wp, 68, h1, h0, w3, w4, w0, w1, w2, 3); + schedule_rounds4!(wp, 72, h0, h1, w4, w0, w1, w2, w3, 3); + schedule_rounds4!(wp, 76, h1, h0, w0, w1, w2, w3, w4, 3); + + let state_abcd = _mm_add_epi32(state_abcd, h0); + let state_e = _mm_sha1nexte_epu32(h1, state_e); + + let state_abcd = _mm_shuffle_epi32(state_abcd, REVERSE); + _mm_storeu_si128(state.as_mut_ptr().cast(), state_abcd); + state[4] = _mm_extract_epi32(state_e, 3) as u32; +} diff --git a/sha1-checked/src/compress/soft.rs b/sha1-checked/src/compress/soft.rs new file mode 100644 index 000000000..924fad91e --- /dev/null +++ b/sha1-checked/src/compress/soft.rs @@ -0,0 +1,662 @@ +//! Direct translation of the C code found at +//! [sha1.c](https://github.com/cr-marcstevens/sha1collisiondetection/blob/master/lib/sha1.c). +//! +//! For the original license and source details see the comments in `src/checked.rs`. + +#![allow(clippy::many_single_char_names, clippy::too_many_arguments)] + +use crate::ubc_check::Testt; + +pub(crate) const K: [u32; 4] = [0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6]; + +#[inline(always)] +fn mix(w: &mut [u32; 80], t: usize) -> u32 { + (w[t - 3] ^ w[t - 8] ^ w[t - 14] ^ w[t - 16]).rotate_left(1) +} + +#[inline(always)] +fn f1(b: u32, c: u32, d: u32) -> u32 { + d ^ b & (c ^ d) +} + +#[inline(always)] +fn f2(b: u32, c: u32, d: u32) -> u32 { + b ^ c ^ d +} + +#[inline(always)] +fn f3(b: u32, c: u32, d: u32) -> u32 { + (b & c).wrapping_add(d & (b ^ c)) +} + +#[inline(always)] +fn f4(b: u32, c: u32, d: u32) -> u32 { + b ^ c ^ d +} + +#[inline(always)] +fn round3_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { + *e = e.wrapping_add( + a.rotate_left(5) + .wrapping_add(f3(*b, c, d)) + .wrapping_add(K[2]) + .wrapping_add(mt), + ); + *b = b.rotate_left(30); +} + +#[inline(always)] +fn round4_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { + *e = e.wrapping_add( + a.rotate_left(5) + .wrapping_add(f4(*b, c, d)) + .wrapping_add(K[3]) + .wrapping_add(mt), + ); + *b = b.rotate_left(30); +} + +#[inline(always)] +fn round1_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { + *b = b.rotate_right(30); + *e = e.wrapping_sub( + a.rotate_left(5) + .wrapping_add(f1(*b, c, d)) + .wrapping_add(K[0]) + .wrapping_add(mt), + ); +} + +#[inline(always)] +fn round2_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { + *b = b.rotate_right(30); + *e = e.wrapping_sub( + a.rotate_left(5) + .wrapping_add(f2(*b, c, d)) + .wrapping_add(K[1]) + .wrapping_add(mt), + ); +} + +#[inline(always)] +fn round3_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { + *b = b.rotate_right(30); + *e = e.wrapping_sub( + a.rotate_left(5) + .wrapping_add(f3(*b, c, d)) + .wrapping_add(K[2]) + .wrapping_add(mt), + ); +} + +#[inline(always)] +fn round4_step_bw(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, mt: u32) { + *b = b.rotate_right(30); + *e = e.wrapping_sub( + a.rotate_left(5) + .wrapping_add(f4(*b, c, d)) + .wrapping_add(K[3]) + .wrapping_add(mt), + ); +} + +#[inline(always)] +fn full_round3_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, w: &mut [u32; 80], t: usize) { + w[t] = mix(w, t); + *e = e.wrapping_add( + w[t].wrapping_add(a.rotate_left(5)) + .wrapping_add(f3(*b, c, d)) + .wrapping_add(K[2]), + ); + *b = b.rotate_left(30); +} + +#[inline(always)] +fn full_round4_step(a: u32, b: &mut u32, c: u32, d: u32, e: &mut u32, w: &mut [u32; 80], t: usize) { + w[t] = mix(w, t); + *e = e.wrapping_add( + w[t].wrapping_add(a.rotate_left(5)) + .wrapping_add(f4(*b, c, d)) + .wrapping_add(K[3]), + ); + *b = b.rotate_left(30); +} + +#[inline] +fn round2_step4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &[u32; 80], + t: usize, +) { + // 1 + *e = e.wrapping_add( + w[t].wrapping_add(a.rotate_left(5)) + .wrapping_add(f2(*b, *c, *d)) + .wrapping_add(K[1]), + ); + *b = b.rotate_left(30); + + // 2 + *d = d.wrapping_add( + w[t + 1] + .wrapping_add(e.rotate_left(5)) + .wrapping_add(f2(*a, *b, *c)) + .wrapping_add(K[1]), + ); + *a = a.rotate_left(30); + + // 3 + *c = c.wrapping_add( + w[t + 2] + .wrapping_add(d.rotate_left(5)) + .wrapping_add(f2(*e, *a, *b)) + .wrapping_add(K[1]), + ); + *e = e.rotate_left(30); + + // 4 + *b = b.wrapping_add( + w[t + 3] + .wrapping_add(c.rotate_left(5)) + .wrapping_add(f2(*d, *e, *a)) + .wrapping_add(K[1]), + ); + *d = d.rotate_left(30); +} + +#[inline] +fn round3_step4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &[u32; 80], + t: usize, +) { + // 1 + *e = e.wrapping_add( + w[t].wrapping_add(a.rotate_left(5)) + .wrapping_add(f3(*b, *c, *d)) + .wrapping_add(K[2]), + ); + *b = b.rotate_left(30); + + // 2 + *d = d.wrapping_add( + w[t + 1] + .wrapping_add(e.rotate_left(5)) + .wrapping_add(f3(*a, *b, *c)) + .wrapping_add(K[2]), + ); + *a = a.rotate_left(30); + + // 3 + *c = c.wrapping_add( + w[t + 2] + .wrapping_add(d.rotate_left(5)) + .wrapping_add(f3(*e, *a, *b)) + .wrapping_add(K[2]), + ); + *e = e.rotate_left(30); + + // 4 + *b = b.wrapping_add( + w[t + 3] + .wrapping_add(c.rotate_left(5)) + .wrapping_add(f3(*d, *e, *a)) + .wrapping_add(K[2]), + ); + *d = d.rotate_left(30); +} + +#[inline] +fn round4_step4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &[u32; 80], + t: usize, +) { + // 1 + *e = e.wrapping_add( + w[t].wrapping_add(a.rotate_left(5)) + .wrapping_add(f4(*b, *c, *d)) + .wrapping_add(K[3]), + ); + *b = b.rotate_left(30); + + // 2 + *d = d.wrapping_add( + w[t + 1] + .wrapping_add(e.rotate_left(5)) + .wrapping_add(f4(*a, *b, *c)) + .wrapping_add(K[3]), + ); + *a = a.rotate_left(30); + + // 3 + *c = c.wrapping_add( + w[t + 2] + .wrapping_add(d.rotate_left(5)) + .wrapping_add(f4(*e, *a, *b)) + .wrapping_add(K[3]), + ); + *e = e.rotate_left(30); + + // 4 + *b = b.wrapping_add( + w[t + 3] + .wrapping_add(c.rotate_left(5)) + .wrapping_add(f4(*d, *e, *a)) + .wrapping_add(K[3]), + ); + *d = d.rotate_left(30); +} + +#[inline] +fn full_round1_step_load4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + m: &[u32; 16], + w: &mut [u32; 80], + t: usize, +) { + // load + w[t..t + 4].copy_from_slice(&m[t..t + 4]); + round1_step4(a, b, c, d, e, w, t); +} + +#[inline(always)] +fn round1_step4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &[u32; 80], + t: usize, +) { + // 1 + *e = e.wrapping_add( + w[t].wrapping_add(a.rotate_left(5)) + .wrapping_add(f1(*b, *c, *d)) + .wrapping_add(K[0]), + ); + *b = b.rotate_left(30); + + // 2 + *d = d.wrapping_add( + w[t + 1] + .wrapping_add(e.rotate_left(5)) + .wrapping_add(f1(*a, *b, *c)) + .wrapping_add(K[0]), + ); + *a = a.rotate_left(30); + + // 3 + *c = c.wrapping_add( + w[t + 2] + .wrapping_add(d.rotate_left(5)) + .wrapping_add(f1(*e, *a, *b)) + .wrapping_add(K[0]), + ); + *e = e.rotate_left(30); + + // 4 + *b = b.wrapping_add( + w[t + 3] + .wrapping_add(c.rotate_left(5)) + .wrapping_add(f1(*d, *e, *a)) + .wrapping_add(K[0]), + ); + *d = d.rotate_left(30); +} + +#[inline] +fn full_round1_step_expand4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &mut [u32; 80], + t: usize, +) { + w[t] = mix(w, t); + w[t + 1] = mix(w, t + 1); + w[t + 2] = mix(w, t + 2); + w[t + 3] = mix(w, t + 3); + round1_step4(a, b, c, d, e, w, t); +} + +#[inline] +fn full_round2_step4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &mut [u32; 80], + t: usize, +) { + w[t] = mix(w, t); + w[t + 1] = mix(w, t + 1); + w[t + 2] = mix(w, t + 2); + w[t + 3] = mix(w, t + 3); + round2_step4(a, b, c, d, e, w, t); +} + +#[inline] +fn full_round3_step4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &mut [u32; 80], + t: usize, +) { + w[t] = mix(w, t); + w[t + 1] = mix(w, t + 1); + w[t + 2] = mix(w, t + 2); + w[t + 3] = mix(w, t + 3); + round3_step4(a, b, c, d, e, w, t); +} + +#[inline] +fn full_round4_step4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + w: &mut [u32; 80], + t: usize, +) { + w[t] = mix(w, t); + w[t + 1] = mix(w, t + 1); + w[t + 2] = mix(w, t + 2); + w[t + 3] = mix(w, t + 3); + round4_step4(a, b, c, d, e, w, t); +} + +#[inline] +fn round1_step_bw4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + m: &[u32; 80], + t: usize, +) { + round1_step_bw(*a, b, *c, *d, e, m[t]); + round1_step_bw(*b, c, *d, *e, a, m[t - 1]); + round1_step_bw(*c, d, *e, *a, b, m[t - 2]); + round1_step_bw(*d, e, *a, *b, c, m[t - 3]); +} + +#[inline] +fn round2_step_bw4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + m: &[u32; 80], + t: usize, +) { + round2_step_bw(*a, b, *c, *d, e, m[t]); + round2_step_bw(*b, c, *d, *e, a, m[t - 1]); + round2_step_bw(*c, d, *e, *a, b, m[t - 2]); + round2_step_bw(*d, e, *a, *b, c, m[t - 3]); +} + +#[inline] +fn round3_step_bw4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + m: &[u32; 80], + t: usize, +) { + round3_step_bw(*a, b, *c, *d, e, m[t]); + round3_step_bw(*b, c, *d, *e, a, m[t - 1]); + round3_step_bw(*c, d, *e, *a, b, m[t - 2]); + round3_step_bw(*d, e, *a, *b, c, m[t - 3]); +} + +#[inline] +fn round4_step_bw4( + a: &mut u32, + b: &mut u32, + c: &mut u32, + d: &mut u32, + e: &mut u32, + m: &[u32; 80], + t: usize, +) { + round4_step_bw(*a, b, *c, *d, e, m[t]); + round4_step_bw(*b, c, *d, *e, a, m[t - 1]); + round4_step_bw(*c, d, *e, *a, b, m[t - 2]); + round4_step_bw(*d, e, *a, *b, c, m[t - 3]); +} + +fn add_assign(left: &mut [u32; 5], right: [u32; 5]) { + left[0] = left[0].wrapping_add(right[0]); + left[1] = left[1].wrapping_add(right[1]); + left[2] = left[2].wrapping_add(right[2]); + left[3] = left[3].wrapping_add(right[3]); + left[4] = left[4].wrapping_add(right[4]); +} + +pub(crate) fn compression_w(ihv: &mut [u32; 5], w: &[u32; 80]) { + let &mut [mut a, mut b, mut c, mut d, mut e] = ihv; + + round1_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 0); + round1_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 4); + round1_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 8); + round1_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 12); + round1_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 16); + + round2_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 20); + round2_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 24); + round2_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 28); + round2_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 32); + round2_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 36); + + round3_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 40); + round3_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 44); + round3_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 48); + round3_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 52); + round3_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 56); + + round4_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 60); + round4_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 64); + round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 68); + round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 72); + round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 76); + + add_assign(ihv, [a, b, c, d, e]); +} + +pub(crate) fn compression_states( + ihv: &mut [u32; 5], + m: &[u32; 16], + w: &mut [u32; 80], + state_58: &mut [u32; 5], + state_65: &mut [u32; 5], +) { + let &mut [mut a, mut b, mut c, mut d, mut e] = ihv; + + full_round1_step_load4(&mut a, &mut b, &mut c, &mut d, &mut e, m, w, 0); + full_round1_step_load4(&mut b, &mut c, &mut d, &mut e, &mut a, m, w, 4); + full_round1_step_load4(&mut c, &mut d, &mut e, &mut a, &mut b, m, w, 8); + full_round1_step_load4(&mut d, &mut e, &mut a, &mut b, &mut c, m, w, 12); + + full_round1_step_expand4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 16); + + full_round2_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 20); + full_round2_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 24); + full_round2_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 28); + full_round2_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 32); + full_round2_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 36); + + full_round3_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 40); + full_round3_step4(&mut b, &mut c, &mut d, &mut e, &mut a, w, 44); + full_round3_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 48); + full_round3_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 52); + + full_round3_step(e, &mut a, b, c, &mut d, w, 56); + full_round3_step(d, &mut e, a, b, &mut c, w, 57); + + // Store state58 + *state_58 = [a, b, c, d, e]; + + full_round3_step(c, &mut d, e, a, &mut b, w, 58); + full_round3_step(b, &mut c, d, e, &mut a, w, 59); + + full_round4_step4(&mut a, &mut b, &mut c, &mut d, &mut e, w, 60); + full_round4_step(b, &mut c, d, e, &mut a, w, 64); + + // Store state65 + *state_65 = [a, b, c, d, e]; + + full_round4_step(a, &mut b, c, d, &mut e, w, 65); + full_round4_step(e, &mut a, b, c, &mut d, w, 66); + full_round4_step(d, &mut e, a, b, &mut c, w, 67); + + full_round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, w, 68); + full_round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, w, 72); + full_round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, w, 76); + + add_assign(ihv, [a, b, c, d, e]); +} + +fn recompress_fast_58( + ihvin: &mut [u32; 5], + ihvout: &mut [u32; 5], + me2: &[u32; 80], + state: &[u32; 5], +) { + let &[mut a, mut b, mut c, mut d, mut e] = state; + + round3_step_bw(d, &mut e, a, b, &mut c, me2[57]); + round3_step_bw(e, &mut a, b, c, &mut d, me2[56]); + + round3_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 55); + round3_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 51); + round3_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 47); + round3_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 43); + + round2_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 39); + round2_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 35); + round2_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 31); + round2_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 27); + round2_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 23); + + round1_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 19); + round1_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 15); + round1_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 11); + round1_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 7); + round1_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 3); + + *ihvin = [a, b, c, d, e]; + [a, b, c, d, e] = *state; + + round3_step(c, &mut d, e, a, &mut b, me2[58]); + round3_step(b, &mut c, d, e, &mut a, me2[59]); + + round4_step4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 60); + round4_step4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 64); + round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 68); + round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 72); + round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 76); + + ihvout[0] = ihvin[0].wrapping_add(a); + ihvout[1] = ihvin[1].wrapping_add(b); + ihvout[2] = ihvin[2].wrapping_add(c); + ihvout[3] = ihvin[3].wrapping_add(d); + ihvout[4] = ihvin[4].wrapping_add(e); +} + +fn recompress_fast_65( + ihvin: &mut [u32; 5], + ihvout: &mut [u32; 5], + me2: &[u32; 80], + state: &[u32; 5], +) { + let &[mut a, mut b, mut c, mut d, mut e] = state; + + round4_step_bw(b, &mut c, d, e, &mut a, me2[64]); + round4_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 63); + + round3_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 59); + round3_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 55); + round3_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 51); + round3_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 47); + round3_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 43); + + round2_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 39); + round2_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 35); + round2_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 31); + round2_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 27); + round2_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 23); + + round1_step_bw4(&mut b, &mut c, &mut d, &mut e, &mut a, me2, 19); + round1_step_bw4(&mut a, &mut b, &mut c, &mut d, &mut e, me2, 15); + round1_step_bw4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 11); + round1_step_bw4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 7); + round1_step_bw4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 3); + + *ihvin = [a, b, c, d, e]; + [a, b, c, d, e] = *state; + + round4_step(a, &mut b, c, d, &mut e, me2[65]); + round4_step(e, &mut a, b, c, &mut d, me2[66]); + round4_step(d, &mut e, a, b, &mut c, me2[67]); + + round4_step4(&mut c, &mut d, &mut e, &mut a, &mut b, me2, 68); + round4_step4(&mut d, &mut e, &mut a, &mut b, &mut c, me2, 72); + round4_step4(&mut e, &mut a, &mut b, &mut c, &mut d, me2, 76); + + ihvout[0] = ihvin[0].wrapping_add(a); + ihvout[1] = ihvin[1].wrapping_add(b); + ihvout[2] = ihvin[2].wrapping_add(c); + ihvout[3] = ihvin[3].wrapping_add(d); + ihvout[4] = ihvin[4].wrapping_add(e); +} + +pub(crate) fn recompression_step( + step: Testt, + ihvin: &mut [u32; 5], + ihvout: &mut [u32; 5], + me2: &[u32; 80], + state: &[u32; 5], +) { + match step { + Testt::T58 => { + recompress_fast_58(ihvin, ihvout, me2, state); + } + Testt::T65 => { + recompress_fast_65(ihvin, ihvout, me2, state); + } + } +} + +#[inline(always)] +pub(crate) fn xor(a: &[u32; 5], b: &[u32; 5]) -> u32 { + a[0] ^ b[0] | a[1] ^ b[1] | a[2] ^ b[2] | a[3] ^ b[3] | a[4] ^ b[4] +}