diff --git a/sha2/src/sha256/riscv_zknh.rs b/sha2/src/sha256/riscv_zknh.rs index bad22ecae..98677e862 100644 --- a/sha2/src/sha256/riscv_zknh.rs +++ b/sha2/src/sha256/riscv_zknh.rs @@ -9,22 +9,9 @@ use core::arch::riscv64::{sha256sig0, sha256sig1, sha256sum0, sha256sum1}; cfg_if::cfg_if! { if #[cfg(sha2_backend_riscv_zknh = "compact")] { mod compact; - use compact::compress_block; + pub(super) use compact::compress; } else { mod unroll; - use unroll::compress_block; + pub(super) use unroll::compress; } } - -#[target_feature(enable = "zknh")] -pub(super) fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) { - for block in blocks.iter().map(load_block) { - compress_block(state, block); - } -} - -fn load_block(block: &[u8; 64]) -> [u32; 16] { - let (chunks, tail) = block.as_chunks(); - assert!(tail.is_empty()); - core::array::from_fn(|i| u32::from_be_bytes(chunks[i])) -} diff --git a/sha2/src/sha256/riscv_zknh/compact.rs b/sha2/src/sha256/riscv_zknh/compact.rs index caa19f91c..e946c41e8 100644 --- a/sha2/src/sha256/riscv_zknh/compact.rs +++ b/sha2/src/sha256/riscv_zknh/compact.rs @@ -1,55 +1,77 @@ use super::{sha256sig0, sha256sig1, sha256sum0, sha256sum1}; use crate::consts::K32; -#[inline] +const ROUNDS: usize = K32.len(); + #[target_feature(enable = "zknh")] -pub(super) fn compress_block(state: &mut [u32; 8], mut block: [u32; 16]) { - let mut s = *state; - - for r in 0..64 { - round(&mut s, &block, r); - if r < 48 { - schedule(&mut block, r) - } - } +pub(crate) fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) { + // Unforuntately, LLVM is unable to remove `memset` for + // `fn message_schedule(block: &[u8; 64]) -> [u32; ROUNDS] { ... }` + // See: https://rust.godbolt.org/z/s61sP3GKj + // So to prevent unnecessary `memset` call for each block we reuse the buffer. + let mut msg = [0u32; ROUNDS]; - for i in 0..8 { - state[i] = state[i].wrapping_add(s[i]); + for block in blocks { + // Pre-computing the whole message schedule instead of doing it during each round + // with a ring buffer results in a smaller binary with less branching + // at the cost of using additional stack space + schedule(block, &mut msg); + rounds(state, &msg); } } #[inline] #[target_feature(enable = "zknh")] -fn schedule(block: &mut [u32; 16], r: usize) { - block[r % 16] = block[r % 16] - .wrapping_add(sha256sig1(block[(r + 14) % 16])) - .wrapping_add(block[(r + 9) % 16]) - .wrapping_add(sha256sig0(block[(r + 1) % 16])); +fn schedule(block: &[u8; 64], msg: &mut [u32; ROUNDS]) { + let (block_chunks, tail) = block.as_chunks(); + assert!(tail.is_empty()); + + let n = block_chunks.len(); + for i in 0..n { + msg[i] = u32::from_be_bytes(block_chunks[i]); + } + + for i in n..ROUNDS { + msg[i] = msg[i - 16] + .wrapping_add(sha256sig0(msg[i - 15])) + .wrapping_add(msg[i - 7]) + .wrapping_add(sha256sig1(msg[i - 2])); + } } #[inline] #[target_feature(enable = "zknh")] -fn round(state: &mut [u32; 8], block: &[u32; 16], r: usize) { - let n = K32.len() - r; - let [a, b, c, d, e, f, g, h] = core::array::from_fn(|i| (n + i) % 8); - - state[h] = state[h] - .wrapping_add(sha256sum1(state[e])) - .wrapping_add(ch(state[e], state[f], state[g])) - .wrapping_add(K32[r]) - .wrapping_add(block[r % 16]); - state[d] = state[d].wrapping_add(state[h]); - state[h] = state[h] - .wrapping_add(sha256sum0(state[a])) - .wrapping_add(maj(state[a], state[b], state[c])) -} +fn rounds(state: &mut [u32; 8], msg: &[u32; ROUNDS]) { + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = *state; -#[inline(always)] -fn ch(x: u32, y: u32, z: u32) -> u32 { - (x & y) ^ (!x & z) -} + for i in 0..ROUNDS { + let s1 = sha256sum1(e); + let ch = (e & f) ^ ((!e) & g); + let t1 = s1 + .wrapping_add(ch) + .wrapping_add(K32[i]) + .wrapping_add(msg[i]) + .wrapping_add(h); + let s0 = sha256sum0(a); + let maj = (a & b) ^ (a & c) ^ (b & c); + let t2 = s0.wrapping_add(maj); + + h = g; + g = f; + f = e; + e = d.wrapping_add(t1); + d = c; + c = b; + b = a; + a = t1.wrapping_add(t2); + } -#[inline(always)] -fn maj(x: u32, y: u32, z: u32) -> u32 { - (x & y) ^ (x & z) ^ (y & z) + state[0] = state[0].wrapping_add(a); + state[1] = state[1].wrapping_add(b); + state[2] = state[2].wrapping_add(c); + state[3] = state[3].wrapping_add(d); + state[4] = state[4].wrapping_add(e); + state[5] = state[5].wrapping_add(f); + state[6] = state[6].wrapping_add(g); + state[7] = state[7].wrapping_add(h); } diff --git a/sha2/src/sha256/riscv_zknh/unroll.rs b/sha2/src/sha256/riscv_zknh/unroll.rs index dc3e9dc16..e868c0514 100644 --- a/sha2/src/sha256/riscv_zknh/unroll.rs +++ b/sha2/src/sha256/riscv_zknh/unroll.rs @@ -1,9 +1,22 @@ use super::{sha256sig0, sha256sig1, sha256sum0, sha256sum1}; use crate::consts::K32; +#[target_feature(enable = "zknh")] +pub(crate) fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) { + for block in blocks { + compress_block(state, block); + } +} + #[inline] #[target_feature(enable = "zknh")] -pub(super) fn compress_block(state: &mut [u32; 8], mut block: [u32; 16]) { +fn compress_block(state: &mut [u32; 8], block: &[u8; 64]) { + let mut block: [u32; 16] = { + let (chunks, tail) = block.as_chunks(); + assert!(tail.is_empty()); + core::array::from_fn(|i| u32::from_be_bytes(chunks[i])) + }; + let s = &mut state.clone(); let b = &mut block; diff --git a/sha2/src/sha512/riscv_zknh.rs b/sha2/src/sha512/riscv_zknh.rs index ecb14fd5f..caa63ae4b 100644 --- a/sha2/src/sha512/riscv_zknh.rs +++ b/sha2/src/sha512/riscv_zknh.rs @@ -4,26 +4,13 @@ compile_error!("riscv-zknh backend can be used only on riscv32 and riscv64 targe cfg_if::cfg_if! { if #[cfg(sha2_backend_riscv_zknh = "compact")] { mod compact; - use compact::compress_block; + pub(super) use compact::compress; } else { mod unroll; - use unroll::compress_block; + pub(super) use unroll::compress; } } -#[target_feature(enable = "zknh")] -pub(super) fn compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) { - for block in blocks.iter().map(load_block) { - compress_block(state, block); - } -} - -fn load_block(block: &[u8; 128]) -> [u64; 16] { - let (chunks, tail) = block.as_chunks(); - assert!(tail.is_empty()); - core::array::from_fn(|i| u64::from_be_bytes(chunks[i])) -} - cfg_if::cfg_if! { if #[cfg(target_arch = "riscv64")] { use core::arch::riscv64::{sha512sig0, sha512sig1, sha512sum0, sha512sum1}; diff --git a/sha2/src/sha512/riscv_zknh/compact.rs b/sha2/src/sha512/riscv_zknh/compact.rs index 23b83ae1c..a6b491f44 100644 --- a/sha2/src/sha512/riscv_zknh/compact.rs +++ b/sha2/src/sha512/riscv_zknh/compact.rs @@ -1,55 +1,77 @@ use super::{sha512sig0, sha512sig1, sha512sum0, sha512sum1}; use crate::consts::K64; -#[inline] +const ROUNDS: usize = K64.len(); + #[target_feature(enable = "zknh")] -pub(super) fn compress_block(state: &mut [u64; 8], mut block: [u64; 16]) { - let mut s = *state; - - for r in 0..80 { - round(&mut s, &block, r); - if r < 64 { - schedule(&mut block, r) - } - } +pub(crate) fn compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) { + // Unforuntately, LLVM is unable to remove `memset` for + // `fn message_schedule(block: &[u8; 128]) -> [u64; ROUNDS] { ... }` + // See: https://rust.godbolt.org/z/ncxzYrMPh + // So to prevent unnecessary `memset` call for each block we reuse the buffer. + let mut msg = [0u64; ROUNDS]; - for i in 0..8 { - state[i] = state[i].wrapping_add(s[i]); + for block in blocks { + // Pre-computing the whole message schedule instead of doing it during each round + // with a ring buffer results in a smaller binary with less branching + // at the cost of using additional stack space + schedule(block, &mut msg); + rounds(state, &msg); } } #[inline] #[target_feature(enable = "zknh")] -fn schedule(block: &mut [u64; 16], r: usize) { - block[r % 16] = block[r % 16] - .wrapping_add(sha512sig1(block[(r + 14) % 16])) - .wrapping_add(block[(r + 9) % 16]) - .wrapping_add(sha512sig0(block[(r + 1) % 16])); +fn schedule(block: &[u8; 128], msg: &mut [u64; ROUNDS]) { + let (block_chunks, tail) = block.as_chunks(); + assert!(tail.is_empty()); + + let n = block_chunks.len(); + for i in 0..n { + msg[i] = u64::from_be_bytes(block_chunks[i]); + } + + for i in n..ROUNDS { + msg[i] = msg[i - 16] + .wrapping_add(sha512sig0(msg[i - 15])) + .wrapping_add(msg[i - 7]) + .wrapping_add(sha512sig1(msg[i - 2])); + } } #[inline] #[target_feature(enable = "zknh")] -fn round(state: &mut [u64; 8], block: &[u64; 16], r: usize) { - let n = K64.len() - r; - let [a, b, c, d, e, f, g, h] = core::array::from_fn(|i| (n + i) % 8); - - state[h] = state[h] - .wrapping_add(sha512sum1(state[e])) - .wrapping_add(ch(state[e], state[f], state[g])) - .wrapping_add(K64[r]) - .wrapping_add(block[r % 16]); - state[d] = state[d].wrapping_add(state[h]); - state[h] = state[h] - .wrapping_add(sha512sum0(state[a])) - .wrapping_add(maj(state[a], state[b], state[c])) -} +fn rounds(state: &mut [u64; 8], msg: &[u64; ROUNDS]) { + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = *state; -#[inline(always)] -fn ch(x: u64, y: u64, z: u64) -> u64 { - (x & y) ^ (!x & z) -} + for i in 0..ROUNDS { + let s1 = sha512sum1(e); + let ch = (e & f) ^ ((!e) & g); + let t1 = s1 + .wrapping_add(ch) + .wrapping_add(K64[i]) + .wrapping_add(msg[i]) + .wrapping_add(h); + let s0 = sha512sum0(a); + let maj = (a & b) ^ (a & c) ^ (b & c); + let t2 = s0.wrapping_add(maj); + + h = g; + g = f; + f = e; + e = d.wrapping_add(t1); + d = c; + c = b; + b = a; + a = t1.wrapping_add(t2); + } -#[inline(always)] -fn maj(x: u64, y: u64, z: u64) -> u64 { - (x & y) ^ (x & z) ^ (y & z) + state[0] = state[0].wrapping_add(a); + state[1] = state[1].wrapping_add(b); + state[2] = state[2].wrapping_add(c); + state[3] = state[3].wrapping_add(d); + state[4] = state[4].wrapping_add(e); + state[5] = state[5].wrapping_add(f); + state[6] = state[6].wrapping_add(g); + state[7] = state[7].wrapping_add(h); } diff --git a/sha2/src/sha512/riscv_zknh/unroll.rs b/sha2/src/sha512/riscv_zknh/unroll.rs index 8aad2038a..56f40e52e 100644 --- a/sha2/src/sha512/riscv_zknh/unroll.rs +++ b/sha2/src/sha512/riscv_zknh/unroll.rs @@ -1,9 +1,22 @@ use super::{sha512sig0, sha512sig1, sha512sum0, sha512sum1}; use crate::consts::K64; +#[target_feature(enable = "zknh")] +pub(crate) fn compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) { + for block in blocks { + compress_block(state, block); + } +} + #[inline] #[target_feature(enable = "zknh")] -pub(super) fn compress_block(state: &mut [u64; 8], mut block: [u64; 16]) { +fn compress_block(state: &mut [u64; 8], block: &[u8; 128]) { + let mut block: [u64; 16] = { + let (chunks, tail) = block.as_chunks(); + assert!(tail.is_empty()); + core::array::from_fn(|i| u64::from_be_bytes(chunks[i])) + }; + let s = &mut state.clone(); let b = &mut block;