From 756e3205200fa5a77c94905771e168cf4d32e575 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 11 Sep 2026 17:25:30 +0000 Subject: [PATCH 1/9] Add runtime-selected LtHash backend with AVX-512 Blake3 XOF kernel --- .github/workflows/lthash-bench.yml | 88 +++ sei-db/state_db/sc/flatkv/lthash/backend.go | 62 ++ .../sc/flatkv/lthash/backend_default.go | 73 +++ .../sc/flatkv/lthash/backend_nosimd.go | 8 + .../sc/flatkv/lthash/backend_simd_amd64.go | 164 ++++++ .../state_db/sc/flatkv/lthash/backend_test.go | 177 ++++++ .../sc/flatkv/lthash/blake3_xof16_amd64.go | 555 ++++++++++++++++++ .../sc/flatkv/lthash/gen_blake3_xof16.go | 118 ++++ sei-db/state_db/sc/flatkv/lthash/lthash.go | 51 +- 9 files changed, 1248 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/lthash-bench.yml create mode 100644 sei-db/state_db/sc/flatkv/lthash/backend.go create mode 100644 sei-db/state_db/sc/flatkv/lthash/backend_default.go create mode 100644 sei-db/state_db/sc/flatkv/lthash/backend_nosimd.go create mode 100644 sei-db/state_db/sc/flatkv/lthash/backend_simd_amd64.go create mode 100644 sei-db/state_db/sc/flatkv/lthash/backend_test.go create mode 100644 sei-db/state_db/sc/flatkv/lthash/blake3_xof16_amd64.go create mode 100644 sei-db/state_db/sc/flatkv/lthash/gen_blake3_xof16.go diff --git a/.github/workflows/lthash-bench.yml b/.github/workflows/lthash-bench.yml new file mode 100644 index 0000000000..aadeade7aa --- /dev/null +++ b/.github/workflows/lthash-bench.yml @@ -0,0 +1,88 @@ +name: LtHash backends +on: + workflow_dispatch: + pull_request: + paths: + - 'sei-db/state_db/sc/flatkv/lthash/**' + - '.github/workflows/lthash-bench.yml' + push: + branches: + - main + paths: + - 'sei-db/state_db/sc/flatkv/lthash/**' + +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + +env: + GO_VERSION: '1.27.1' + LTHASH_PKG: ./sei-db/state_db/sc/flatkv/lthash + BENCH_COUNT: 8 + +jobs: + bench: + name: Default vs SIMD + runs-on: uci-default + steps: + # See: https://github.com/actions/checkout/releases/tag/v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 1 + + - uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Download modules + run: go mod download + + - name: CPU + run: | + { + echo '## CPU' + echo '```' + lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + # Both builds must pass: the default build has no SIMD backend compiled + # in, the experiment build adds it and the differential tests compare + # every available backend against the Blake3 reference. + - name: Test without GOEXPERIMENT=simd + run: go test -count=1 -race ${{ env.LTHASH_PKG }} + + - name: Test with GOEXPERIMENT=simd + env: + GOEXPERIMENT: simd + run: go test -count=1 -race ${{ env.LTHASH_PKG }} + + - name: Benchmark every available backend + env: + GOEXPERIMENT: simd + run: | + go test \ + -run '^$' \ + -bench . \ + -count ${{ env.BENCH_COUNT }} \ + ${{ env.LTHASH_PKG }} | tee bench.txt + + - name: Compare backends + run: | + go install golang.org/x/perf/cmd/benchstat@v0.0.0-20250813145418-2f7363a06fe1 + { + echo '## LtHash backends (`benchstat -col /backend`)' + echo + echo 'Only backends the runner CPU can execute appear as columns; the SIMD' + echo 'backend needs AVX-512F + VBMI2. `hashChunk` is the end-to-end per-block path.' + echo + echo '```' + benchstat -col /backend bench.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + with: + name: lthash-bench + path: bench.txt diff --git a/sei-db/state_db/sc/flatkv/lthash/backend.go b/sei-db/state_db/sc/flatkv/lthash/backend.go new file mode 100644 index 0000000000..58c4819893 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/lthash/backend.go @@ -0,0 +1,62 @@ +package lthash + +import ( + "os" + "sort" +) + +// BackendEnv is the environment variable that pins the hashing backend by +// name. An unknown or empty value leaves the selection automatic. +const BackendEnv = "SEI_LTHASH_BACKEND" + +// backend is one implementation of the LtHash primitives. Every backend must +// produce bit-identical results; they differ only in how fast they get there. +type backend struct { + name string + // expand fills dst with the 2048-byte Blake3 XOF of data, one + // little-endian uint16 per limb. data is never empty. + expand func(data []byte, dst *LtHash) + // add and sub are element-wise mod 2^16 on the limb vectors. + add func(dst, src *LtHash) + sub func(dst, src *LtHash) +} + +var active = selectBackend(os.Getenv(BackendEnv)) + +// ActiveBackend returns the name of the hashing backend in use. +func ActiveBackend() string { + return active.name +} + +// availableBackends returns every backend this binary can run on this CPU, +// keyed by name. +func availableBackends() map[string]backend { + m := map[string]backend{defaultBackend.name: defaultBackend} + if b, ok := simdBackend(); ok { + m[b.name] = b + } + return m +} + +// availableBackendNames returns the names from availableBackends, sorted. +func availableBackendNames() []string { + m := availableBackends() + names := make([]string, 0, len(m)) + for name := range m { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// selectBackend picks the backend named by pin, or the fastest available one +// when pin is empty or unknown. +func selectBackend(pin string) backend { + if b, ok := availableBackends()[pin]; ok { + return b + } + if b, ok := simdBackend(); ok { + return b + } + return defaultBackend +} diff --git a/sei-db/state_db/sc/flatkv/lthash/backend_default.go b/sei-db/state_db/sc/flatkv/lthash/backend_default.go new file mode 100644 index 0000000000..20ff55d757 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/lthash/backend_default.go @@ -0,0 +1,73 @@ +package lthash + +import ( + "encoding/binary" + "sync" + + "github.com/zeebo/blake3" +) + +// defaultBackend is the portable implementation: zeebo/blake3 for the XOF and +// plain Go loops for the limb arithmetic. It is always compiled in. +var defaultBackend = backend{ + name: "default", + expand: expandBlake3, + add: addScalar, + sub: subScalar, +} + +func expandBlake3(data []byte, dst *LtHash) { + hasher := blake3HasherPool.Get().(*blake3.Hasher) + hasher.Reset() + _, _ = hasher.Write(data) + digest := hasher.Digest() + + bufPtr := xofBufferPool.Get().(*[]byte) + output := *bufPtr + _, _ = digest.Read(output) // Blake3 XOF never errors and always fills buffer + blake3HasherPool.Put(hasher) + + for i := 0; i < LtHashSize; i++ { + dst.limbs[i] = binary.LittleEndian.Uint16(output[i*2 : (i+1)*2]) + } + xofBufferPool.Put(bufPtr) +} + +func addScalar(dst, src *LtHash) { + for i := 0; i < LtHashSize; i += 8 { + dst.limbs[i] += src.limbs[i] + dst.limbs[i+1] += src.limbs[i+1] + dst.limbs[i+2] += src.limbs[i+2] + dst.limbs[i+3] += src.limbs[i+3] + dst.limbs[i+4] += src.limbs[i+4] + dst.limbs[i+5] += src.limbs[i+5] + dst.limbs[i+6] += src.limbs[i+6] + dst.limbs[i+7] += src.limbs[i+7] + } +} + +func subScalar(dst, src *LtHash) { + for i := 0; i < LtHashSize; i += 8 { + dst.limbs[i] -= src.limbs[i] + dst.limbs[i+1] -= src.limbs[i+1] + dst.limbs[i+2] -= src.limbs[i+2] + dst.limbs[i+3] -= src.limbs[i+3] + dst.limbs[i+4] -= src.limbs[i+4] + dst.limbs[i+5] -= src.limbs[i+5] + dst.limbs[i+6] -= src.limbs[i+6] + dst.limbs[i+7] -= src.limbs[i+7] + } +} + +var xofBufferPool = sync.Pool{ + New: func() interface{} { + buf := make([]byte, LtHashBytes) + return &buf + }, +} + +var blake3HasherPool = sync.Pool{ + New: func() interface{} { + return blake3.New() + }, +} diff --git a/sei-db/state_db/sc/flatkv/lthash/backend_nosimd.go b/sei-db/state_db/sc/flatkv/lthash/backend_nosimd.go new file mode 100644 index 0000000000..8d71dce513 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/lthash/backend_nosimd.go @@ -0,0 +1,8 @@ +//go:build !(goexperiment.simd && amd64) + +package lthash + +// simdBackend reports that no SIMD backend is compiled into this binary. +func simdBackend() (backend, bool) { + return backend{}, false +} diff --git a/sei-db/state_db/sc/flatkv/lthash/backend_simd_amd64.go b/sei-db/state_db/sc/flatkv/lthash/backend_simd_amd64.go new file mode 100644 index 0000000000..8c689b5caf --- /dev/null +++ b/sei-db/state_db/sc/flatkv/lthash/backend_simd_amd64.go @@ -0,0 +1,164 @@ +//go:build goexperiment.simd && amd64 + +package lthash + +import ( + "encoding/binary" + "math/bits" + "unsafe" + + "simd/archsimd" +) + +//go:generate go run gen_blake3_xof16.go + +// simdBackendName is the name reported by ActiveBackend for the AVX-512 path. +const simdBackendName = "simd" + +// simdBackend returns the AVX-512 backend when the CPU can run it. The XOF +// kernel needs AVX-512F (Uint32x16) and VBMI2 (VPSHRDD rotates); the limb +// arithmetic needs AVX-512BW (Uint16x32). +func simdBackend() (backend, bool) { + if !archsimd.X86.AVX512() || !archsimd.X86.AVX512VBMI2() { + return backend{}, false + } + return backend{ + name: simdBackendName, + expand: expandSIMD, + add: addSIMD, + sub: subSIMD, + }, true +} + +const ( + blake3BlockLen = 64 + blake3ChunkLen = 1024 + + blake3ChunkStart = 1 << 0 + blake3ChunkEnd = 1 << 1 + blake3Root = 1 << 3 +) + +var blake3IV = [8]uint32{ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +} + +// expandSIMD computes the 2048-byte Blake3 XOF of data as two 16-lane root +// compressions. Inputs longer than one chunk need the Blake3 tree, which the +// default backend already implements. +func expandSIMD(data []byte, dst *LtHash) { + if len(data) > blake3ChunkLen { + expandBlake3(data, dst) + return + } + var in xof16Inputs + singleChunkRoot(data, &in) + out := (*[2][16][16]uint32)(unsafe.Pointer(&dst.limbs)) //nolint:gosec // G103: same size, little-endian limb layout + xof16(&in, &out[0]) + for lane := range in[12] { + in[12][lane] = 16 + } + xof16(&in, &out[1]) +} + +// singleChunkRoot compresses all but the last block of a one-chunk message +// into a chaining value and broadcasts the root compression inputs into in. +func singleChunkRoot(data []byte, in *xof16Inputs) { + cv := blake3IV + flags := uint32(blake3ChunkStart) + var block [16]uint32 + for len(data) > blake3BlockLen { + loadBlock(&block, data[:blake3BlockLen]) + out := blake3Compress(&cv, &block, 0, blake3BlockLen, flags) + copy(cv[:], out[:8]) + flags = 0 + data = data[blake3BlockLen:] + } + var last [blake3BlockLen]byte + copy(last[:], data) + loadBlock(&block, last[:]) + flags |= blake3ChunkEnd | blake3Root + + for lane := 0; lane < 16; lane++ { + for i := 0; i < 8; i++ { + in[i][lane] = cv[i] + in[8+i][lane] = blake3IV[i] + } + in[12][lane] = 0 + in[13][lane] = 0 + in[14][lane] = uint32(len(data)) //nolint:gosec // G115: len(data) <= blake3BlockLen + in[15][lane] = flags + for i := 0; i < 16; i++ { + in[16+i][lane] = block[i] + } + } +} + +func loadBlock(block *[16]uint32, b []byte) { + for i := range block { + block[i] = binary.LittleEndian.Uint32(b[4*i:]) + } +} + +// blake3Compress is the scalar Blake3 compression function, returning the +// full 16-word state (only the first 8 words are the chaining value). +func blake3Compress(cv *[8]uint32, block *[16]uint32, counter uint64, blockLen, flags uint32) [16]uint32 { + s := [16]uint32{ + cv[0], cv[1], cv[2], cv[3], cv[4], cv[5], cv[6], cv[7], + blake3IV[0], blake3IV[1], blake3IV[2], blake3IV[3], + uint32(counter), uint32(counter >> 32), blockLen, flags, //nolint:gosec // G115: counter is split into its two 32-bit halves + } + m := *block + for r := 0; r < 7; r++ { + blake3G(&s, 0, 4, 8, 12, m[0], m[1]) + blake3G(&s, 1, 5, 9, 13, m[2], m[3]) + blake3G(&s, 2, 6, 10, 14, m[4], m[5]) + blake3G(&s, 3, 7, 11, 15, m[6], m[7]) + blake3G(&s, 0, 5, 10, 15, m[8], m[9]) + blake3G(&s, 1, 6, 11, 12, m[10], m[11]) + blake3G(&s, 2, 7, 8, 13, m[12], m[13]) + blake3G(&s, 3, 4, 9, 14, m[14], m[15]) + m = [16]uint32{ + m[2], m[6], m[3], m[10], m[7], m[0], m[4], m[13], + m[1], m[11], m[12], m[5], m[9], m[14], m[15], m[8], + } + } + for i := 0; i < 8; i++ { + s[i] ^= s[i+8] + s[i+8] ^= cv[i] + } + return s +} + +func blake3G(s *[16]uint32, a, b, c, d int, mx, my uint32) { + s[a] += s[b] + mx + s[d] = bits.RotateLeft32(s[d]^s[a], -16) + s[c] += s[d] + s[b] = bits.RotateLeft32(s[b]^s[c], -12) + s[a] += s[b] + my + s[d] = bits.RotateLeft32(s[d]^s[a], -8) + s[c] += s[d] + s[b] = bits.RotateLeft32(s[b]^s[c], -7) +} + +const simdLimbVectors = LtHashSize / 32 + +// limbVectors views the limbs as 32-lane vectors; the sizes are identical. +func limbVectors(l *LtHash) *[simdLimbVectors][32]uint16 { + return (*[simdLimbVectors][32]uint16)(unsafe.Pointer(&l.limbs)) //nolint:gosec // G103 +} + +func addSIMD(dst, src *LtHash) { + a, b := limbVectors(dst), limbVectors(src) + for i := range a { + archsimd.LoadUint16x32Array(&a[i]).Add(archsimd.LoadUint16x32Array(&b[i])).StoreArray(&a[i]) + } +} + +func subSIMD(dst, src *LtHash) { + a, b := limbVectors(dst), limbVectors(src) + for i := range a { + archsimd.LoadUint16x32Array(&a[i]).Sub(archsimd.LoadUint16x32Array(&b[i])).StoreArray(&a[i]) + } +} diff --git a/sei-db/state_db/sc/flatkv/lthash/backend_test.go b/sei-db/state_db/sc/flatkv/lthash/backend_test.go new file mode 100644 index 0000000000..4a7a81e20c --- /dev/null +++ b/sei-db/state_db/sc/flatkv/lthash/backend_test.go @@ -0,0 +1,177 @@ +package lthash + +import ( + "encoding/binary" + "fmt" + "math/rand" + "testing" + + "github.com/zeebo/blake3" +) + +// referenceExpand is the specification every backend must match: the first +// 2048 bytes of the Blake3 XOF, read as little-endian uint16 limbs. +func referenceExpand(data []byte) *LtHash { + var out [LtHashBytes]byte + h := blake3.New() + _, _ = h.Write(data) + _, _ = h.Digest().Read(out[:]) + lth := New() + for i := range lth.limbs { + lth.limbs[i] = binary.LittleEndian.Uint16(out[2*i:]) + } + return lth +} + +// expandSizes covers block and chunk boundaries of Blake3, including the +// multi-chunk tree path that the SIMD backend delegates. +var expandSizes = []int{1, 8, 63, 64, 65, 124, 127, 128, 129, 500, 1023, 1024, 1025, 2048, 4096, 5000} + +func TestBackendsAgreeWithReference(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + for name, b := range availableBackends() { + t.Run(name, func(t *testing.T) { + for _, n := range expandSizes { + for iter := 0; iter < 8; iter++ { + data := make([]byte, n) + rng.Read(data) + got := New() + b.expand(data, got) + if want := referenceExpand(data); !got.Equal(want) { + t.Fatalf("expand(%d bytes) differs from the Blake3 reference", n) + } + } + } + }) + } +} + +func TestBackendsAgreeOnMix(t *testing.T) { + rng := rand.New(rand.NewSource(2)) + x, y := New(), New() + for i := range x.limbs { + x.limbs[i] = uint16(rng.Uint32()) + y.limbs[i] = uint16(rng.Uint32()) + } + wantAdd, wantSub := x.Clone(), x.Clone() + addScalar(wantAdd, y) + subScalar(wantSub, y) + for name, b := range availableBackends() { + t.Run(name, func(t *testing.T) { + gotAdd, gotSub := x.Clone(), x.Clone() + b.add(gotAdd, y) + b.sub(gotSub, y) + if !gotAdd.Equal(wantAdd) { + t.Fatal("add differs from scalar") + } + if !gotSub.Equal(wantSub) { + t.Fatal("sub differs from scalar") + } + }) + } +} + +func TestSelectBackend(t *testing.T) { + if got := selectBackend("default").name; got != "default" { + t.Fatalf("pinning default selected %q", got) + } + want := "default" + if simd, ok := simdBackend(); ok { + want = simd.name + } + if got := selectBackend("").name; got != want { + t.Fatalf("automatic selection picked %q, want %q", got, want) + } + if got := selectBackend("no-such-backend").name; got != selectBackend("").name { + t.Fatalf("unknown pin %q should fall back to automatic selection", got) + } + if _, ok := availableBackends()[ActiveBackend()]; !ok { + t.Fatalf("active backend %q is not available", ActiveBackend()) + } +} + +// Benchmarks are keyed by backend so `benchstat -col /backend` places the +// implementations side by side. + +func benchmarkKV() []byte { + rng := rand.New(rand.NewSource(3)) + key := make([]byte, 40) + value := make([]byte, 76) + rng.Read(key) + rng.Read(value) + return serializeKV(key, value) +} + +func forEachBackend(b *testing.B, fn func(b *testing.B, be backend)) { + all := availableBackends() + for _, name := range availableBackendNames() { + be := all[name] + b.Run(fmt.Sprintf("backend=%s", name), func(b *testing.B) { fn(b, be) }) + } +} + +func BenchmarkExpand(b *testing.B) { + data := benchmarkKV() + forEachBackend(b, func(b *testing.B, be backend) { + dst := New() + b.SetBytes(LtHashBytes) + for i := 0; i < b.N; i++ { + be.expand(data, dst) + } + }) +} + +func BenchmarkMixIn(b *testing.B) { + forEachBackend(b, func(b *testing.B, be backend) { + x, y := New(), New() + for i := 0; i < b.N; i++ { + be.add(x, y) + } + }) +} + +func BenchmarkMixOut(b *testing.B) { + forEachBackend(b, func(b *testing.B, be backend) { + x, y := New(), New() + for i := 0; i < b.N; i++ { + be.sub(x, y) + } + }) +} + +// BenchmarkHashKV is one leaf update as hashChunk performs it: expand the +// serialized pair and fold it into an accumulator. +func BenchmarkHashKV(b *testing.B) { + data := benchmarkKV() + forEachBackend(b, func(b *testing.B, be backend) { + acc, h := New(), New() + for i := 0; i < b.N; i++ { + be.expand(data, h) + be.add(acc, h) + } + }) +} + +// BenchmarkHashChunk runs the full mutation pipeline through the active +// backend, switching the active backend for each sub-benchmark. +func BenchmarkHashChunk(b *testing.B) { + rng := rand.New(rand.NewSource(4)) + mutations := make([]KeyMutation, 1000) + for i := range mutations { + key := make([]byte, 40) + last := make([]byte, 76) + value := make([]byte, 76) + rng.Read(key) + rng.Read(last) + rng.Read(value) + mutations[i] = KeyMutation{Key: key, LastValue: last, Value: value} + } + saved := active + defer func() { active = saved }() + forEachBackend(b, func(b *testing.B, be backend) { + active = be + for i := 0; i < b.N; i++ { + hashChunk(mutations) + } + }) +} diff --git a/sei-db/state_db/sc/flatkv/lthash/blake3_xof16_amd64.go b/sei-db/state_db/sc/flatkv/lthash/blake3_xof16_amd64.go new file mode 100644 index 0000000000..487a138138 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/lthash/blake3_xof16_amd64.go @@ -0,0 +1,555 @@ +// Code generated by gen_blake3_xof16.go; DO NOT EDIT. + +//go:build goexperiment.simd && amd64 + +package lthash + +import "simd/archsimd" + +// xof16Inputs holds the 16-lane broadcast of every compression input. +// Rows 0-7 are the chaining value, 8-11 the IV, 12 the block counter base, +// 13 zero (counter high word), 14 the block length, 15 the flags and 16-31 the +// message words. Loading a pre-broadcast row is used instead of +// archsimd.BroadcastUint32x16, which compiles to a legacy-SSE sequence that +// costs an SSE/AVX transition on every call. +type xof16Inputs [32][16]uint32 + +var xof16Lanes = [16]uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + +// rotr32 rotates every lane right by n using VPSHRDD; RotateAllRight is +// emulated with three instructions. +func rotr32(x archsimd.Uint32x16, n uint64) archsimd.Uint32x16 { + return x.ShiftAllRightConcatMod32(x, n) +} + +// xof16 writes output blocks base..base+15 of the root XOF to out, +// out[lane][word]. +func xof16(in *xof16Inputs, out *[16][16]uint32) { + s0 := archsimd.LoadUint32x16Array(&in[0]) + s1 := archsimd.LoadUint32x16Array(&in[1]) + s2 := archsimd.LoadUint32x16Array(&in[2]) + s3 := archsimd.LoadUint32x16Array(&in[3]) + s4 := archsimd.LoadUint32x16Array(&in[4]) + s5 := archsimd.LoadUint32x16Array(&in[5]) + s6 := archsimd.LoadUint32x16Array(&in[6]) + s7 := archsimd.LoadUint32x16Array(&in[7]) + s8 := archsimd.LoadUint32x16Array(&in[8]) + s9 := archsimd.LoadUint32x16Array(&in[9]) + s10 := archsimd.LoadUint32x16Array(&in[10]) + s11 := archsimd.LoadUint32x16Array(&in[11]) + s12 := archsimd.LoadUint32x16Array(&xof16Lanes).Add(archsimd.LoadUint32x16Array(&in[12])) + s13 := archsimd.LoadUint32x16Array(&in[13]) + s14 := archsimd.LoadUint32x16Array(&in[14]) + s15 := archsimd.LoadUint32x16Array(&in[15]) + m0 := archsimd.LoadUint32x16Array(&in[16]) + m1 := archsimd.LoadUint32x16Array(&in[17]) + m2 := archsimd.LoadUint32x16Array(&in[18]) + m3 := archsimd.LoadUint32x16Array(&in[19]) + m4 := archsimd.LoadUint32x16Array(&in[20]) + m5 := archsimd.LoadUint32x16Array(&in[21]) + m6 := archsimd.LoadUint32x16Array(&in[22]) + m7 := archsimd.LoadUint32x16Array(&in[23]) + m8 := archsimd.LoadUint32x16Array(&in[24]) + m9 := archsimd.LoadUint32x16Array(&in[25]) + m10 := archsimd.LoadUint32x16Array(&in[26]) + m11 := archsimd.LoadUint32x16Array(&in[27]) + m12 := archsimd.LoadUint32x16Array(&in[28]) + m13 := archsimd.LoadUint32x16Array(&in[29]) + m14 := archsimd.LoadUint32x16Array(&in[30]) + m15 := archsimd.LoadUint32x16Array(&in[31]) + // round 0 + s0 = s0.Add(s4).Add(m0) + s12 = rotr32(s12.Xor(s0), 16) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 12) + s0 = s0.Add(s4).Add(m1) + s12 = rotr32(s12.Xor(s0), 8) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 7) + s1 = s1.Add(s5).Add(m2) + s13 = rotr32(s13.Xor(s1), 16) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 12) + s1 = s1.Add(s5).Add(m3) + s13 = rotr32(s13.Xor(s1), 8) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 7) + s2 = s2.Add(s6).Add(m4) + s14 = rotr32(s14.Xor(s2), 16) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 12) + s2 = s2.Add(s6).Add(m5) + s14 = rotr32(s14.Xor(s2), 8) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 7) + s3 = s3.Add(s7).Add(m6) + s15 = rotr32(s15.Xor(s3), 16) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 12) + s3 = s3.Add(s7).Add(m7) + s15 = rotr32(s15.Xor(s3), 8) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 7) + s0 = s0.Add(s5).Add(m8) + s15 = rotr32(s15.Xor(s0), 16) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 12) + s0 = s0.Add(s5).Add(m9) + s15 = rotr32(s15.Xor(s0), 8) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 7) + s1 = s1.Add(s6).Add(m10) + s12 = rotr32(s12.Xor(s1), 16) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 12) + s1 = s1.Add(s6).Add(m11) + s12 = rotr32(s12.Xor(s1), 8) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 7) + s2 = s2.Add(s7).Add(m12) + s13 = rotr32(s13.Xor(s2), 16) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 12) + s2 = s2.Add(s7).Add(m13) + s13 = rotr32(s13.Xor(s2), 8) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 7) + s3 = s3.Add(s4).Add(m14) + s14 = rotr32(s14.Xor(s3), 16) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 12) + s3 = s3.Add(s4).Add(m15) + s14 = rotr32(s14.Xor(s3), 8) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 7) + // round 1 + s0 = s0.Add(s4).Add(m2) + s12 = rotr32(s12.Xor(s0), 16) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 12) + s0 = s0.Add(s4).Add(m6) + s12 = rotr32(s12.Xor(s0), 8) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 7) + s1 = s1.Add(s5).Add(m3) + s13 = rotr32(s13.Xor(s1), 16) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 12) + s1 = s1.Add(s5).Add(m10) + s13 = rotr32(s13.Xor(s1), 8) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 7) + s2 = s2.Add(s6).Add(m7) + s14 = rotr32(s14.Xor(s2), 16) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 12) + s2 = s2.Add(s6).Add(m0) + s14 = rotr32(s14.Xor(s2), 8) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 7) + s3 = s3.Add(s7).Add(m4) + s15 = rotr32(s15.Xor(s3), 16) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 12) + s3 = s3.Add(s7).Add(m13) + s15 = rotr32(s15.Xor(s3), 8) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 7) + s0 = s0.Add(s5).Add(m1) + s15 = rotr32(s15.Xor(s0), 16) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 12) + s0 = s0.Add(s5).Add(m11) + s15 = rotr32(s15.Xor(s0), 8) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 7) + s1 = s1.Add(s6).Add(m12) + s12 = rotr32(s12.Xor(s1), 16) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 12) + s1 = s1.Add(s6).Add(m5) + s12 = rotr32(s12.Xor(s1), 8) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 7) + s2 = s2.Add(s7).Add(m9) + s13 = rotr32(s13.Xor(s2), 16) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 12) + s2 = s2.Add(s7).Add(m14) + s13 = rotr32(s13.Xor(s2), 8) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 7) + s3 = s3.Add(s4).Add(m15) + s14 = rotr32(s14.Xor(s3), 16) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 12) + s3 = s3.Add(s4).Add(m8) + s14 = rotr32(s14.Xor(s3), 8) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 7) + // round 2 + s0 = s0.Add(s4).Add(m3) + s12 = rotr32(s12.Xor(s0), 16) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 12) + s0 = s0.Add(s4).Add(m4) + s12 = rotr32(s12.Xor(s0), 8) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 7) + s1 = s1.Add(s5).Add(m10) + s13 = rotr32(s13.Xor(s1), 16) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 12) + s1 = s1.Add(s5).Add(m12) + s13 = rotr32(s13.Xor(s1), 8) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 7) + s2 = s2.Add(s6).Add(m13) + s14 = rotr32(s14.Xor(s2), 16) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 12) + s2 = s2.Add(s6).Add(m2) + s14 = rotr32(s14.Xor(s2), 8) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 7) + s3 = s3.Add(s7).Add(m7) + s15 = rotr32(s15.Xor(s3), 16) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 12) + s3 = s3.Add(s7).Add(m14) + s15 = rotr32(s15.Xor(s3), 8) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 7) + s0 = s0.Add(s5).Add(m6) + s15 = rotr32(s15.Xor(s0), 16) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 12) + s0 = s0.Add(s5).Add(m5) + s15 = rotr32(s15.Xor(s0), 8) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 7) + s1 = s1.Add(s6).Add(m9) + s12 = rotr32(s12.Xor(s1), 16) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 12) + s1 = s1.Add(s6).Add(m0) + s12 = rotr32(s12.Xor(s1), 8) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 7) + s2 = s2.Add(s7).Add(m11) + s13 = rotr32(s13.Xor(s2), 16) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 12) + s2 = s2.Add(s7).Add(m15) + s13 = rotr32(s13.Xor(s2), 8) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 7) + s3 = s3.Add(s4).Add(m8) + s14 = rotr32(s14.Xor(s3), 16) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 12) + s3 = s3.Add(s4).Add(m1) + s14 = rotr32(s14.Xor(s3), 8) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 7) + // round 3 + s0 = s0.Add(s4).Add(m10) + s12 = rotr32(s12.Xor(s0), 16) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 12) + s0 = s0.Add(s4).Add(m7) + s12 = rotr32(s12.Xor(s0), 8) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 7) + s1 = s1.Add(s5).Add(m12) + s13 = rotr32(s13.Xor(s1), 16) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 12) + s1 = s1.Add(s5).Add(m9) + s13 = rotr32(s13.Xor(s1), 8) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 7) + s2 = s2.Add(s6).Add(m14) + s14 = rotr32(s14.Xor(s2), 16) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 12) + s2 = s2.Add(s6).Add(m3) + s14 = rotr32(s14.Xor(s2), 8) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 7) + s3 = s3.Add(s7).Add(m13) + s15 = rotr32(s15.Xor(s3), 16) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 12) + s3 = s3.Add(s7).Add(m15) + s15 = rotr32(s15.Xor(s3), 8) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 7) + s0 = s0.Add(s5).Add(m4) + s15 = rotr32(s15.Xor(s0), 16) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 12) + s0 = s0.Add(s5).Add(m0) + s15 = rotr32(s15.Xor(s0), 8) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 7) + s1 = s1.Add(s6).Add(m11) + s12 = rotr32(s12.Xor(s1), 16) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 12) + s1 = s1.Add(s6).Add(m2) + s12 = rotr32(s12.Xor(s1), 8) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 7) + s2 = s2.Add(s7).Add(m5) + s13 = rotr32(s13.Xor(s2), 16) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 12) + s2 = s2.Add(s7).Add(m8) + s13 = rotr32(s13.Xor(s2), 8) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 7) + s3 = s3.Add(s4).Add(m1) + s14 = rotr32(s14.Xor(s3), 16) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 12) + s3 = s3.Add(s4).Add(m6) + s14 = rotr32(s14.Xor(s3), 8) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 7) + // round 4 + s0 = s0.Add(s4).Add(m12) + s12 = rotr32(s12.Xor(s0), 16) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 12) + s0 = s0.Add(s4).Add(m13) + s12 = rotr32(s12.Xor(s0), 8) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 7) + s1 = s1.Add(s5).Add(m9) + s13 = rotr32(s13.Xor(s1), 16) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 12) + s1 = s1.Add(s5).Add(m11) + s13 = rotr32(s13.Xor(s1), 8) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 7) + s2 = s2.Add(s6).Add(m15) + s14 = rotr32(s14.Xor(s2), 16) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 12) + s2 = s2.Add(s6).Add(m10) + s14 = rotr32(s14.Xor(s2), 8) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 7) + s3 = s3.Add(s7).Add(m14) + s15 = rotr32(s15.Xor(s3), 16) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 12) + s3 = s3.Add(s7).Add(m8) + s15 = rotr32(s15.Xor(s3), 8) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 7) + s0 = s0.Add(s5).Add(m7) + s15 = rotr32(s15.Xor(s0), 16) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 12) + s0 = s0.Add(s5).Add(m2) + s15 = rotr32(s15.Xor(s0), 8) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 7) + s1 = s1.Add(s6).Add(m5) + s12 = rotr32(s12.Xor(s1), 16) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 12) + s1 = s1.Add(s6).Add(m3) + s12 = rotr32(s12.Xor(s1), 8) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 7) + s2 = s2.Add(s7).Add(m0) + s13 = rotr32(s13.Xor(s2), 16) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 12) + s2 = s2.Add(s7).Add(m1) + s13 = rotr32(s13.Xor(s2), 8) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 7) + s3 = s3.Add(s4).Add(m6) + s14 = rotr32(s14.Xor(s3), 16) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 12) + s3 = s3.Add(s4).Add(m4) + s14 = rotr32(s14.Xor(s3), 8) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 7) + // round 5 + s0 = s0.Add(s4).Add(m9) + s12 = rotr32(s12.Xor(s0), 16) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 12) + s0 = s0.Add(s4).Add(m14) + s12 = rotr32(s12.Xor(s0), 8) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 7) + s1 = s1.Add(s5).Add(m11) + s13 = rotr32(s13.Xor(s1), 16) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 12) + s1 = s1.Add(s5).Add(m5) + s13 = rotr32(s13.Xor(s1), 8) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 7) + s2 = s2.Add(s6).Add(m8) + s14 = rotr32(s14.Xor(s2), 16) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 12) + s2 = s2.Add(s6).Add(m12) + s14 = rotr32(s14.Xor(s2), 8) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 7) + s3 = s3.Add(s7).Add(m15) + s15 = rotr32(s15.Xor(s3), 16) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 12) + s3 = s3.Add(s7).Add(m1) + s15 = rotr32(s15.Xor(s3), 8) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 7) + s0 = s0.Add(s5).Add(m13) + s15 = rotr32(s15.Xor(s0), 16) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 12) + s0 = s0.Add(s5).Add(m3) + s15 = rotr32(s15.Xor(s0), 8) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 7) + s1 = s1.Add(s6).Add(m0) + s12 = rotr32(s12.Xor(s1), 16) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 12) + s1 = s1.Add(s6).Add(m10) + s12 = rotr32(s12.Xor(s1), 8) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 7) + s2 = s2.Add(s7).Add(m2) + s13 = rotr32(s13.Xor(s2), 16) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 12) + s2 = s2.Add(s7).Add(m6) + s13 = rotr32(s13.Xor(s2), 8) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 7) + s3 = s3.Add(s4).Add(m4) + s14 = rotr32(s14.Xor(s3), 16) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 12) + s3 = s3.Add(s4).Add(m7) + s14 = rotr32(s14.Xor(s3), 8) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 7) + // round 6 + s0 = s0.Add(s4).Add(m11) + s12 = rotr32(s12.Xor(s0), 16) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 12) + s0 = s0.Add(s4).Add(m15) + s12 = rotr32(s12.Xor(s0), 8) + s8 = s8.Add(s12) + s4 = rotr32(s4.Xor(s8), 7) + s1 = s1.Add(s5).Add(m5) + s13 = rotr32(s13.Xor(s1), 16) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 12) + s1 = s1.Add(s5).Add(m0) + s13 = rotr32(s13.Xor(s1), 8) + s9 = s9.Add(s13) + s5 = rotr32(s5.Xor(s9), 7) + s2 = s2.Add(s6).Add(m1) + s14 = rotr32(s14.Xor(s2), 16) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 12) + s2 = s2.Add(s6).Add(m9) + s14 = rotr32(s14.Xor(s2), 8) + s10 = s10.Add(s14) + s6 = rotr32(s6.Xor(s10), 7) + s3 = s3.Add(s7).Add(m8) + s15 = rotr32(s15.Xor(s3), 16) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 12) + s3 = s3.Add(s7).Add(m6) + s15 = rotr32(s15.Xor(s3), 8) + s11 = s11.Add(s15) + s7 = rotr32(s7.Xor(s11), 7) + s0 = s0.Add(s5).Add(m14) + s15 = rotr32(s15.Xor(s0), 16) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 12) + s0 = s0.Add(s5).Add(m10) + s15 = rotr32(s15.Xor(s0), 8) + s10 = s10.Add(s15) + s5 = rotr32(s5.Xor(s10), 7) + s1 = s1.Add(s6).Add(m2) + s12 = rotr32(s12.Xor(s1), 16) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 12) + s1 = s1.Add(s6).Add(m12) + s12 = rotr32(s12.Xor(s1), 8) + s11 = s11.Add(s12) + s6 = rotr32(s6.Xor(s11), 7) + s2 = s2.Add(s7).Add(m3) + s13 = rotr32(s13.Xor(s2), 16) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 12) + s2 = s2.Add(s7).Add(m4) + s13 = rotr32(s13.Xor(s2), 8) + s8 = s8.Add(s13) + s7 = rotr32(s7.Xor(s8), 7) + s3 = s3.Add(s4).Add(m7) + s14 = rotr32(s14.Xor(s3), 16) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 12) + s3 = s3.Add(s4).Add(m13) + s14 = rotr32(s14.Xor(s3), 8) + s9 = s9.Add(s14) + s4 = rotr32(s4.Xor(s9), 7) + // finalize: full 16-word output for the root XOF + s0 = s0.Xor(s8) + s8 = s8.Xor(archsimd.LoadUint32x16Array(&in[0])) + s1 = s1.Xor(s9) + s9 = s9.Xor(archsimd.LoadUint32x16Array(&in[1])) + s2 = s2.Xor(s10) + s10 = s10.Xor(archsimd.LoadUint32x16Array(&in[2])) + s3 = s3.Xor(s11) + s11 = s11.Xor(archsimd.LoadUint32x16Array(&in[3])) + s4 = s4.Xor(s12) + s12 = s12.Xor(archsimd.LoadUint32x16Array(&in[4])) + s5 = s5.Xor(s13) + s13 = s13.Xor(archsimd.LoadUint32x16Array(&in[5])) + s6 = s6.Xor(s14) + s14 = s14.Xor(archsimd.LoadUint32x16Array(&in[6])) + s7 = s7.Xor(s15) + s15 = s15.Xor(archsimd.LoadUint32x16Array(&in[7])) + // s_j holds word j of every lane; transpose so out[lane] is one block. + var t [16][16]uint32 + s0.StoreArray(&t[0]) + s1.StoreArray(&t[1]) + s2.StoreArray(&t[2]) + s3.StoreArray(&t[3]) + s4.StoreArray(&t[4]) + s5.StoreArray(&t[5]) + s6.StoreArray(&t[6]) + s7.StoreArray(&t[7]) + s8.StoreArray(&t[8]) + s9.StoreArray(&t[9]) + s10.StoreArray(&t[10]) + s11.StoreArray(&t[11]) + s12.StoreArray(&t[12]) + s13.StoreArray(&t[13]) + s14.StoreArray(&t[14]) + s15.StoreArray(&t[15]) + for lane := 0; lane < 16; lane++ { + for word := 0; word < 16; word++ { + out[lane][word] = t[word][lane] + } + } +} diff --git a/sei-db/state_db/sc/flatkv/lthash/gen_blake3_xof16.go b/sei-db/state_db/sc/flatkv/lthash/gen_blake3_xof16.go new file mode 100644 index 0000000000..ba95f11dd0 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/lthash/gen_blake3_xof16.go @@ -0,0 +1,118 @@ +//go:build ignore + +// gen_blake3_xof16 writes blake3_xof16_amd64.go: a fully unrolled 16-lane +// Blake3 root compression over simd/archsimd Uint32x16 vectors. Each lane is one +// output block of the XOF (counter base+lane), so a single call produces +// 16 x 64 bytes of XOF output; two calls produce the 2048 bytes an LtHash needs. +// +// Usage: go run gen_blake3_xof16.go +package main + +import ( + "bytes" + "fmt" + "go/format" + "os" +) + +// msgPerm is the Blake3 message word permutation applied between rounds. +var msgPerm = [16]int{2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8} + +func main() { + var b bytes.Buffer + p := func(format string, args ...any) { fmt.Fprintf(&b, format+"\n", args...) } + + p("// Code generated by gen_blake3_xof16.go; DO NOT EDIT.") + p("") + p("//go:build goexperiment.simd && amd64") + p("") + p("package lthash") + p("") + p(`import "simd/archsimd"`) + p("") + p("// xof16Inputs holds the 16-lane broadcast of every compression input.") + p("// Rows 0-7 are the chaining value, 8-11 the IV, 12 the block counter base,") + p("// 13 zero (counter high word), 14 the block length, 15 the flags and 16-31 the") + p("// message words. Loading a pre-broadcast row is used instead of") + p("// archsimd.BroadcastUint32x16, which compiles to a legacy-SSE sequence that") + p("// costs an SSE/AVX transition on every call.") + p("type xof16Inputs [32][16]uint32") + p("") + p("var xof16Lanes = [16]uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}") + p("") + p("// rotr32 rotates every lane right by n using VPSHRDD; RotateAllRight is") + p("// emulated with three instructions.") + p("func rotr32(x archsimd.Uint32x16, n uint64) archsimd.Uint32x16 {") + p("\treturn x.ShiftAllRightConcatMod32(x, n)") + p("}") + p("") + p("// xof16 writes output blocks base..base+15 of the root XOF to out,") + p("// out[lane][word].") + p("func xof16(in *xof16Inputs, out *[16][16]uint32) {") + for i := 0; i < 12; i++ { + p("\ts%d := archsimd.LoadUint32x16Array(&in[%d])", i, i) + } + p("\ts12 := archsimd.LoadUint32x16Array(&xof16Lanes).Add(archsimd.LoadUint32x16Array(&in[12]))") + for i := 13; i < 16; i++ { + p("\ts%d := archsimd.LoadUint32x16Array(&in[%d])", i, i) + } + for i := 0; i < 16; i++ { + p("\tm%d := archsimd.LoadUint32x16Array(&in[%d])", i, 16+i) + } + g := func(a, b, c, d, mx, my int) { + p("\ts%d = s%d.Add(s%d).Add(m%d)", a, a, b, mx) + p("\ts%d = rotr32(s%d.Xor(s%d), 16)", d, d, a) + p("\ts%d = s%d.Add(s%d)", c, c, d) + p("\ts%d = rotr32(s%d.Xor(s%d), 12)", b, b, c) + p("\ts%d = s%d.Add(s%d).Add(m%d)", a, a, b, my) + p("\ts%d = rotr32(s%d.Xor(s%d), 8)", d, d, a) + p("\ts%d = s%d.Add(s%d)", c, c, d) + p("\ts%d = rotr32(s%d.Xor(s%d), 7)", b, b, c) + } + m := [16]int{} + for i := range m { + m[i] = i + } + for r := 0; r < 7; r++ { + p("\t// round %d", r) + g(0, 4, 8, 12, m[0], m[1]) + g(1, 5, 9, 13, m[2], m[3]) + g(2, 6, 10, 14, m[4], m[5]) + g(3, 7, 11, 15, m[6], m[7]) + g(0, 5, 10, 15, m[8], m[9]) + g(1, 6, 11, 12, m[10], m[11]) + g(2, 7, 8, 13, m[12], m[13]) + g(3, 4, 9, 14, m[14], m[15]) + var next [16]int + for i := range next { + next[i] = m[msgPerm[i]] + } + m = next + } + p("\t// finalize: full 16-word output for the root XOF") + for i := 0; i < 8; i++ { + p("\ts%d = s%d.Xor(s%d)", i, i, i+8) + p("\ts%d = s%d.Xor(archsimd.LoadUint32x16Array(&in[%d]))", i+8, i+8, i) + } + p("\t// s_j holds word j of every lane; transpose so out[lane] is one block.") + p("\tvar t [16][16]uint32") + for j := 0; j < 16; j++ { + p("\ts%d.StoreArray(&t[%d])", j, j) + } + p("\tfor lane := 0; lane < 16; lane++ {") + p("\t\tfor word := 0; word < 16; word++ {") + p("\t\t\tout[lane][word] = t[word][lane]") + p("\t\t}") + p("\t}") + p("}") + + src, err := format.Source(b.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := os.WriteFile("blake3_xof16_amd64.go", src, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/sei-db/state_db/sc/flatkv/lthash/lthash.go b/sei-db/state_db/sc/flatkv/lthash/lthash.go index 47348ee1ae..fcf4131708 100644 --- a/sei-db/state_db/sc/flatkv/lthash/lthash.go +++ b/sei-db/state_db/sc/flatkv/lthash/lthash.go @@ -47,16 +47,7 @@ func (l *LtHash) MixIn(other *LtHash) { if other == nil { return } - for i := 0; i < LtHashSize; i += 8 { - l.limbs[i] += other.limbs[i] - l.limbs[i+1] += other.limbs[i+1] - l.limbs[i+2] += other.limbs[i+2] - l.limbs[i+3] += other.limbs[i+3] - l.limbs[i+4] += other.limbs[i+4] - l.limbs[i+5] += other.limbs[i+5] - l.limbs[i+6] += other.limbs[i+6] - l.limbs[i+7] += other.limbs[i+7] - } + active.add(l, other) } // MixOut subtracts other from this LtHash (element-wise mod 2^16). Nil is a no-op. @@ -64,16 +55,7 @@ func (l *LtHash) MixOut(other *LtHash) { if other == nil { return } - for i := 0; i < LtHashSize; i += 8 { - l.limbs[i] -= other.limbs[i] - l.limbs[i+1] -= other.limbs[i+1] - l.limbs[i+2] -= other.limbs[i+2] - l.limbs[i+3] -= other.limbs[i+3] - l.limbs[i+4] -= other.limbs[i+4] - l.limbs[i+5] -= other.limbs[i+5] - l.limbs[i+6] -= other.limbs[i+6] - l.limbs[i+7] -= other.limbs[i+7] - } + active.sub(l, other) } // Equal returns true if both LtHash vectors are identical. @@ -137,22 +119,8 @@ func hash(data []byte) *LtHash { if len(data) == 0 { return New() } - - hasher := blake3HasherPool.Get().(*blake3.Hasher) - hasher.Reset() - _, _ = hasher.Write(data) - digest := hasher.Digest() - - bufPtr := xofBufferPool.Get().(*[]byte) - output := *bufPtr - _, _ = digest.Read(output) // Blake3 XOF never errors and always fills buffer - blake3HasherPool.Put(hasher) - lth := ltHashPool.Get().(*LtHash) - for i := 0; i < LtHashSize; i++ { - lth.limbs[i] = binary.LittleEndian.Uint16(output[i*2 : (i+1)*2]) - } - xofBufferPool.Put(bufPtr) + active.expand(data, lth) return lth } @@ -183,19 +151,6 @@ func serializeKV(key, value []byte) []byte { // --- internal pools --- -var xofBufferPool = sync.Pool{ - New: func() interface{} { - buf := make([]byte, LtHashBytes) - return &buf - }, -} - -var blake3HasherPool = sync.Pool{ - New: func() interface{} { - return blake3.New() - }, -} - var checksumBufferPool = sync.Pool{ New: func() interface{} { buf := make([]byte, LtHashBytes) From cb9829ed6d6acc4b4fd3fec210de250fa32cfa98 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 11 Sep 2026 17:33:52 +0000 Subject: [PATCH 2/9] Run LtHash benchmark job on several runner pools and flag missing AVX-512 --- .github/workflows/lthash-bench.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lthash-bench.yml b/.github/workflows/lthash-bench.yml index aadeade7aa..2eb9469874 100644 --- a/.github/workflows/lthash-bench.yml +++ b/.github/workflows/lthash-bench.yml @@ -22,8 +22,14 @@ env: jobs: bench: - name: Default vs SIMD - runs-on: uci-default + name: Default vs SIMD (${{ matrix.runner }}) + # Not every runner pool has AVX-512; several are tried so at least one is + # likely to execute the SIMD backend. + strategy: + fail-fast: false + matrix: + runner: [uci-default, ubuntu-latest] + runs-on: ${{ matrix.runner }} steps: # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -41,7 +47,7 @@ jobs: - name: CPU run: | { - echo '## CPU' + echo '## CPU (${{ matrix.runner }})' echo '```' lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 echo '```' @@ -74,15 +80,21 @@ jobs: { echo '## LtHash backends (`benchstat -col /backend`)' echo - echo 'Only backends the runner CPU can execute appear as columns; the SIMD' - echo 'backend needs AVX-512F + VBMI2. `hashChunk` is the end-to-end per-block path.' + if grep -q 'backend=simd' bench.txt; then + echo '`hashChunk` is the end-to-end per-block path.' + else + echo '**This runner CPU lacks AVX-512F + VBMI2, so only the default backend ran.**' + fi echo echo '```' benchstat -col /backend bench.txt echo '```' } >> "$GITHUB_STEP_SUMMARY" + if ! grep -q 'backend=simd' bench.txt; then + echo '::warning::SIMD backend unavailable on this runner CPU; only the default backend was benchmarked' + fi - uses: actions/upload-artifact@v4 with: - name: lthash-bench + name: lthash-bench-${{ matrix.runner }} path: bench.txt From 356d8e31802c6ba5ec09177e00206bd0dc2b546d Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 11 Sep 2026 20:47:52 +0000 Subject: [PATCH 3/9] Summarise LtHash benchmarks with benchstat in log, job summary and PR comment --- .github/workflows/lthash-bench.yml | 49 +++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/.github/workflows/lthash-bench.yml b/.github/workflows/lthash-bench.yml index 2eb9469874..df1bc278bd 100644 --- a/.github/workflows/lthash-bench.yml +++ b/.github/workflows/lthash-bench.yml @@ -15,6 +15,10 @@ concurrency: cancel-in-progress: true group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} +permissions: + contents: read + pull-requests: write + env: GO_VERSION: '1.27.1' LTHASH_PKG: ./sei-db/state_db/sc/flatkv/lthash @@ -46,12 +50,7 @@ jobs: - name: CPU run: | - { - echo '## CPU (${{ matrix.runner }})' - echo '```' - lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 # Both builds must pass: the default build has no SIMD backend compiled # in, the experiment build adds it and the differential tests compare @@ -74,14 +73,19 @@ jobs: -count ${{ env.BENCH_COUNT }} \ ${{ env.LTHASH_PKG }} | tee bench.txt - - name: Compare backends + # benchstat groups the samples by the `backend=` sub-benchmark name, so the + # simd column reads as a delta against default. The report is printed to + # the log, added to the job summary and upserted as a PR comment. + - name: Summarise with benchstat run: | go install golang.org/x/perf/cmd/benchstat@v0.0.0-20250813145418-2f7363a06fe1 { - echo '## LtHash backends (`benchstat -col /backend`)' + echo '### LtHash default vs SIMD (`${{ matrix.runner }}`)' + echo + echo "CPU: $(lscpu | sed -nE 's/^Model name:\s+//p')" echo if grep -q 'backend=simd' bench.txt; then - echo '`hashChunk` is the end-to-end per-block path.' + echo '`HashChunk` is the end-to-end per-block path; `vs base` is simd relative to default.' else echo '**This runner CPU lacks AVX-512F + VBMI2, so only the default backend ran.**' fi @@ -89,12 +93,35 @@ jobs: echo '```' benchstat -col /backend bench.txt echo '```' - } >> "$GITHUB_STEP_SUMMARY" + } | tee benchstat.md >> "$GITHUB_STEP_SUMMARY" if ! grep -q 'backend=simd' bench.txt; then echo '::warning::SIMD backend unavailable on this runner CPU; only the default backend was benchmarked' fi + - name: Post benchstat report on the PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v8 + env: + MARKER: '' + with: + script: | + const fs = require('fs'); + const marker = process.env.MARKER; + const body = `${marker}\n${fs.readFileSync('benchstat.md', 'utf8')}`; + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number, per_page: 100 }); + const existing = comments.find(c => c.body && c.body.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } + - uses: actions/upload-artifact@v4 with: name: lthash-bench-${{ matrix.runner }} - path: bench.txt + path: | + bench.txt + benchstat.md From 39c0b09fb491fb5f9a054c0988f1392dbbc137a8 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 11 Sep 2026 21:47:32 +0000 Subject: [PATCH 4/9] Add runtime-selected SHA-256 batch backend for tendermint Merkle hashing Adds sei-tendermint/crypto/tmhash with a default crypto/sha256 backend and a 16-lane AVX-512 kernel built under goexperiment.simd, and routes merkle.HashFromByteSlices through level-batched hashing when a multi-lane backend is active. Output is byte-identical to the recursive tree. --- .github/workflows/lthash-bench.yml | 105 ++++- sei-tendermint/crypto/merkle/tree.go | 33 ++ sei-tendermint/crypto/merkle/tree_test.go | 59 +++ sei-tendermint/crypto/tmhash/backend.go | 73 ++++ .../crypto/tmhash/backend_default.go | 21 + .../crypto/tmhash/backend_nosimd.go | 7 + .../crypto/tmhash/backend_simd_amd64.go | 267 ++++++++++++ sei-tendermint/crypto/tmhash/backend_test.go | 107 +++++ sei-tendermint/crypto/tmhash/gen_sha256x16.go | 138 ++++++ .../crypto/tmhash/sha256x16_amd64.go | 410 ++++++++++++++++++ .../crypto/tmhash/zeroupper_amd64.s | 8 + 11 files changed, 1226 insertions(+), 2 deletions(-) create mode 100644 sei-tendermint/crypto/tmhash/backend.go create mode 100644 sei-tendermint/crypto/tmhash/backend_default.go create mode 100644 sei-tendermint/crypto/tmhash/backend_nosimd.go create mode 100644 sei-tendermint/crypto/tmhash/backend_simd_amd64.go create mode 100644 sei-tendermint/crypto/tmhash/backend_test.go create mode 100644 sei-tendermint/crypto/tmhash/gen_sha256x16.go create mode 100644 sei-tendermint/crypto/tmhash/sha256x16_amd64.go create mode 100644 sei-tendermint/crypto/tmhash/zeroupper_amd64.s diff --git a/.github/workflows/lthash-bench.yml b/.github/workflows/lthash-bench.yml index df1bc278bd..03b7b78274 100644 --- a/.github/workflows/lthash-bench.yml +++ b/.github/workflows/lthash-bench.yml @@ -1,15 +1,19 @@ -name: LtHash backends +name: SIMD hash backends on: workflow_dispatch: pull_request: paths: - 'sei-db/state_db/sc/flatkv/lthash/**' + - 'sei-tendermint/crypto/tmhash/**' + - 'sei-tendermint/crypto/merkle/**' - '.github/workflows/lthash-bench.yml' push: branches: - main paths: - 'sei-db/state_db/sc/flatkv/lthash/**' + - 'sei-tendermint/crypto/tmhash/**' + - 'sei-tendermint/crypto/merkle/**' concurrency: cancel-in-progress: true @@ -26,7 +30,7 @@ env: jobs: bench: - name: Default vs SIMD (${{ matrix.runner }}) + name: LtHash default vs SIMD (${{ matrix.runner }}) # Not every runner pool has AVX-512; several are tried so at least one is # likely to execute the SIMD backend. strategy: @@ -125,3 +129,100 @@ jobs: path: | bench.txt benchstat.md + + bench-tmhash: + name: tmhash default vs SIMD (${{ matrix.runner }}) + strategy: + fail-fast: false + matrix: + runner: [uci-default, ubuntu-latest] + runs-on: ${{ matrix.runner }} + defaults: + run: + working-directory: sei-tendermint + steps: + # See: https://github.com/actions/checkout/releases/tag/v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 1 + + - uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Download modules + run: go mod download + + - name: CPU + run: | + lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 + + - name: Test without GOEXPERIMENT=simd + run: go test -count=1 -race ./crypto/tmhash ./crypto/merkle + + - name: Test with GOEXPERIMENT=simd + env: + GOEXPERIMENT: simd + run: go test -count=1 -race ./crypto/tmhash ./crypto/merkle + + # The default column comes from a plain build: with GOEXPERIMENT=simd + # on an AVX-512 CPU the runtime's async preemption restores ZMM + # registers without VZEROUPPER, which slows the legacy-SSE SHA-NI path + # and would skew the baseline. + - name: Benchmark every available backend + run: | + { + go test -run '^$' -bench 'SumBatch|HashFromByteSlices' -count ${{ env.BENCH_COUNT }} ./crypto/tmhash ./crypto/merkle + GOEXPERIMENT=simd go test -run '^$' -bench 'SumBatch|HashFromByteSlices' -count ${{ env.BENCH_COUNT }} ./crypto/tmhash ./crypto/merkle + } | tee bench.txt + + - name: Summarise with benchstat + run: | + go install golang.org/x/perf/cmd/benchstat@v0.0.0-20250813145418-2f7363a06fe1 + { + echo '### tmhash / merkle default vs SIMD (`${{ matrix.runner }}`)' + echo + echo "CPU: $(lscpu | sed -nE 's/^Model name:\s+//p')" + echo + if grep -q 'backend=simd' bench.txt; then + echo '`HashFromByteSlices` is the end-to-end Merkle root; `vs base` is simd relative to default (SHA-NI).' + else + echo '**This runner CPU lacks AVX-512F + VBMI + VBMI2, so only the default backend ran.**' + fi + echo + echo '```' + benchstat -col /backend bench.txt + echo '```' + } | tee benchstat.md >> "$GITHUB_STEP_SUMMARY" + if ! grep -q 'backend=simd' bench.txt; then + echo '::warning::SIMD backend unavailable on this runner CPU; only the default backend was benchmarked' + fi + + - name: Post benchstat report on the PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v8 + env: + MARKER: '' + with: + script: | + const fs = require('fs'); + const marker = process.env.MARKER; + const body = `${marker}\n${fs.readFileSync('sei-tendermint/benchstat.md', 'utf8')}`; + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number, per_page: 100 }); + const existing = comments.find(c => c.body && c.body.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } + + - uses: actions/upload-artifact@v4 + with: + name: tmhash-bench-${{ matrix.runner }} + path: | + sei-tendermint/bench.txt + sei-tendermint/benchstat.md diff --git a/sei-tendermint/crypto/merkle/tree.go b/sei-tendermint/crypto/merkle/tree.go index 0dac5d4757..662be86fd3 100644 --- a/sei-tendermint/crypto/merkle/tree.go +++ b/sei-tendermint/crypto/merkle/tree.go @@ -4,14 +4,47 @@ import ( "crypto/sha256" "hash" "math/bits" + + "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/tmhash" ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. It follows RFC-6962. func HashFromByteSlices(items [][]byte) []byte { + if lanes := tmhash.BatchLanes(); lanes > 1 && len(items) >= lanes { + return hashFromByteSlicesBatched(items) + } return hashFromByteSlices(sha256.New(), items) } +// hashFromByteSlicesBatched builds the same tree as hashFromByteSlices one +// level at a time, handing every level's independent hashes to +// tmhash.SumBatch in one call. Pairing adjacent nodes left to right and +// carrying an odd trailing node up unchanged yields the RFC-6962 split. +func hashFromByteSlicesBatched(items [][]byte) []byte { + nodes := make([][tmhash.Size]byte, len(items)) + tmhash.SumBatch(leafPrefix, items, nodes) + pairs := make([]byte, len(items)/2*2*tmhash.Size) + msgs := make([][]byte, len(items)/2) + for len(nodes) > 1 { + np := len(nodes) / 2 + for i := range np { + pair := pairs[i*2*tmhash.Size : (i+1)*2*tmhash.Size] + copy(pair, nodes[2*i][:]) + copy(pair[tmhash.Size:], nodes[2*i+1][:]) + msgs[i] = pair + } + carry := len(nodes) % 2 + if carry == 1 { + nodes[np] = nodes[len(nodes)-1] + } + tmhash.SumBatch(innerPrefix, msgs[:np], nodes[:np]) + nodes = nodes[:np+carry] + } + root := nodes[0] + return root[:] +} + func hashFromByteSlices(sha hash.Hash, items [][]byte) []byte { switch len(items) { case 0: diff --git a/sei-tendermint/crypto/merkle/tree_test.go b/sei-tendermint/crypto/merkle/tree_test.go index d558cc8c9b..2801967598 100644 --- a/sei-tendermint/crypto/merkle/tree_test.go +++ b/sei-tendermint/crypto/merkle/tree_test.go @@ -2,6 +2,7 @@ package merkle import ( "bytes" + "crypto/sha256" "encoding/binary" "encoding/hex" "io" @@ -156,6 +157,64 @@ func BenchmarkHashAlternatives(b *testing.B) { }) } +// TestHashFromByteSlicesBatched checks the level-batched tree against the +// recursive reference around every lane-count boundary. +func TestHashFromByteSlicesBatched(t *testing.T) { + sha := sha256.New() + for _, size := range []int{2, 32, 128} { + for total := 1; total <= 70; total++ { + items := make([][]byte, total) + for i := range items { + items[i] = tmrand.Bytes(size) + } + require.Equal(t, hashFromByteSlices(sha, items), hashFromByteSlicesBatched(items), "size=%d total=%d", size, total) + } + } +} + +// BenchmarkHashFromByteSlices measures the whole tree for tx-hash sized and +// tx sized leaves. The sub-benchmark is named after the active tmhash +// backend so runs under different SEI_TMHASH_BACKEND values can be compared +// with benchstat. +func BenchmarkHashFromByteSlices(b *testing.B) { + for _, tc := range []struct { + name string + total int + size int + }{ + {"leaves=1024/leaf=32", 1024, 32}, + {"leaves=1024/leaf=512", 1024, 512}, + {"leaves=100/leaf=32", 100, 32}, + } { + items := make([][]byte, tc.total) + for i := range items { + items[i] = tmrand.Bytes(tc.size) + } + b.Run(tc.name+"/backend="+tmhash.ActiveBackend(), func(b *testing.B) { + b.SetBytes(int64(tc.total * tc.size)) + for b.Loop() { + _ = HashFromByteSlices(items) + } + }) + } +} + +// BenchmarkHashFromByteSlicesBatched forces the level-batched tree so that, +// pinned to the default backend, it isolates the restructuring from the SIMD +// kernel. +func BenchmarkHashFromByteSlicesBatched(b *testing.B) { + items := make([][]byte, 1024) + for i := range items { + items[i] = tmrand.Bytes(32) + } + b.Run("leaves=1024/leaf=32/backend="+tmhash.ActiveBackend(), func(b *testing.B) { + b.SetBytes(int64(len(items) * 32)) + for b.Loop() { + _ = hashFromByteSlicesBatched(items) + } + }) +} + func Test_getSplitPoint(t *testing.T) { tests := []struct { length int64 diff --git a/sei-tendermint/crypto/tmhash/backend.go b/sei-tendermint/crypto/tmhash/backend.go new file mode 100644 index 0000000000..0e70c415a2 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend.go @@ -0,0 +1,73 @@ +package tmhash + +import ( + "os" + "slices" +) + +// BackendEnv is the environment variable that pins the batch hashing backend +// by name. An unknown or empty value leaves the selection automatic. +const BackendEnv = "SEI_TMHASH_BACKEND" + +// backend is one implementation of batched SHA-256. Every backend produces +// bit-identical digests; they differ only in how fast they get there. +type backend struct { + name string + // lanes is the number of equal-length messages the backend hashes at + // once; 1 means one message at a time. + lanes int + // sumBatch writes SHA-256(prefix || msgs[i]) to out[i] for every i. + sumBatch func(prefix []byte, msgs [][]byte, out [][Size]byte) +} + +var active = selectBackend(os.Getenv(BackendEnv)) + +// ActiveBackend returns the name of the batch hashing backend in use. +func ActiveBackend() string { + return active.name +} + +// BatchLanes returns how many messages the active backend hashes in parallel. +// Callers batching work should hand over multiples of this many messages. +func BatchLanes() int { + return active.lanes +} + +// SumBatch writes SHA-256(prefix || msgs[i]) to out[i] for every i. +// out must be at least as long as msgs. +func SumBatch(prefix []byte, msgs [][]byte, out [][Size]byte) { + active.sumBatch(prefix, msgs, out) +} + +// availableBackends returns every backend this binary can run on this CPU, +// keyed by name. +func availableBackends() map[string]backend { + m := map[string]backend{defaultBackend.name: defaultBackend} + if b, ok := simdBackend(); ok { + m[b.name] = b + } + return m +} + +// availableBackendNames returns the names from availableBackends, sorted. +func availableBackendNames() []string { + m := availableBackends() + names := make([]string, 0, len(m)) + for name := range m { + names = append(names, name) + } + slices.Sort(names) + return names +} + +// selectBackend picks the backend named by pin, or the fastest available one +// when pin is empty or unknown. +func selectBackend(pin string) backend { + if b, ok := availableBackends()[pin]; ok { + return b + } + if b, ok := simdBackend(); ok { + return b + } + return defaultBackend +} diff --git a/sei-tendermint/crypto/tmhash/backend_default.go b/sei-tendermint/crypto/tmhash/backend_default.go new file mode 100644 index 0000000000..33767ed6dd --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_default.go @@ -0,0 +1,21 @@ +package tmhash + +import "crypto/sha256" + +// defaultBackend hashes one message at a time with crypto/sha256, which uses +// the SHA-NI single-lane instructions where the CPU has them. +var defaultBackend = backend{ + name: "default", + lanes: 1, + sumBatch: sumBatchScalar, +} + +func sumBatchScalar(prefix []byte, msgs [][]byte, out [][Size]byte) { + h := sha256.New() + for i, msg := range msgs { + h.Reset() + h.Write(prefix) + h.Write(msg) + h.Sum(out[i][:0]) + } +} diff --git a/sei-tendermint/crypto/tmhash/backend_nosimd.go b/sei-tendermint/crypto/tmhash/backend_nosimd.go new file mode 100644 index 0000000000..aa4ceec377 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_nosimd.go @@ -0,0 +1,7 @@ +//go:build !(goexperiment.simd && amd64) + +package tmhash + +func simdBackend() (backend, bool) { + return backend{}, false +} diff --git a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go new file mode 100644 index 0000000000..d715438484 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go @@ -0,0 +1,267 @@ +//go:build goexperiment.simd && amd64 + +package tmhash + +import ( + "crypto/sha256" + "encoding/binary" + "sync" + + "simd/archsimd" +) + +//go:generate go run gen_sha256x16.go + +// vzeroupper clears the upper halves of the vector registers. The compiler +// does not emit it after AVX-512 code, and legacy-SSE code that follows (the +// SHA-NI scalar path, memmove) runs several times slower while they are dirty. +// +//go:noescape +func vzeroupper() + +const ( + simdBackendName = "simd" + simdLanes = 16 + blockSize = 64 + // simdMaxBlocks bounds the per-lane message length hashed by the SIMD + // kernel; longer messages go through the scalar path so the block + // scratch stays small. + simdMaxBlocks = 64 +) + +// simdBackend returns the AVX-512 backend when the CPU can run it. AVX512 +// covers F/BW/DQ/VL; VBMI provides VPERMB for the prefix shift and VBMI2 +// VPSHRDD for the rotates. +func simdBackend() (backend, bool) { + if !archsimd.X86.AVX512() || !archsimd.X86.AVX512VBMI() || !archsimd.X86.AVX512VBMI2() { + return backend{}, false + } + return backend{ + name: simdBackendName, + lanes: simdLanes, + sumBatch: sumBatchSIMD, + }, true +} + +// laneScratch is the per-call working set: the transposed message blocks fed +// to the kernel and, per lane, the final padding block's big-endian bit length +// in bytes 56-63 (the other bytes stay zero). +type laneScratch struct { + blocks []sha256Block16 + lens [simdLanes][blockSize]byte + pfx prefixShift +} + +func (sp *laneScratch) setPrefix(prefix []byte) { + if len(prefix) > 0 { + sp.pfx.init(prefix) + } +} + +var laneScratchPool = sync.Pool{ + New: func() any { return &laneScratch{blocks: make([]sha256Block16, 4)} }, +} + +// paddedBlocks returns how many 64-byte blocks SHA-256 processes for a message +// of n bytes: the message, a 0x80 byte and a 64-bit length. +func paddedBlocks(n int) int { + return (n + 1 + 8 + blockSize - 1) / blockSize +} + +// sumBatchSIMD hashes msgs sixteen at a time. Lanes must share a block count, +// so messages are bucketed by padded length; buckets with fewer than sixteen +// messages left over fall back to the scalar backend. +func sumBatchSIMD(prefix []byte, msgs [][]byte, out [][Size]byte) { + if len(msgs) < simdLanes || len(prefix) >= blockSize { + sumBatchScalar(prefix, msgs, out) + return + } + sp := laneScratchPool.Get().(*laneScratch) + sp.setPrefix(prefix) + // Bucket message indices by block count. Merkle levels are either all + // inner nodes or leaves of similar size, so most calls stay on the + // single-bucket path. + var buckets map[int][]int + var rest []int + var lanes [simdLanes]int + first := paddedBlocks(len(prefix) + len(msgs[0])) + next := 0 + for i, msg := range msgs { + nb := paddedBlocks(len(prefix) + len(msg)) + if nb == first && nb <= simdMaxBlocks { + lanes[next] = i + if next++; next == simdLanes { + sha256Lanes(sp, prefix, msgs, out, &lanes, nb) + next = 0 + } + continue + } + if nb > simdMaxBlocks { + rest = append(rest, i) + continue + } + if buckets == nil { + buckets = map[int][]int{} + } + buckets[nb] = append(buckets[nb], i) + } + rest = append(rest, lanes[:next]...) + for nb, idx := range buckets { + for len(idx) >= simdLanes { + copy(lanes[:], idx[:simdLanes]) + idx = idx[simdLanes:] + sha256Lanes(sp, prefix, msgs, out, &lanes, nb) + } + rest = append(rest, idx...) + } + laneScratchPool.Put(sp) + vzeroupper() + if len(rest) > 0 { + h := sha256.New() + for _, i := range rest { + h.Reset() + h.Write(prefix) + h.Write(msgs[i]) + h.Sum(out[i][:0]) + } + } +} + +// sha256Lanes hashes the sixteen messages selected by lanes, each of nb +// padded blocks, with one kernel call. +func sha256Lanes(sp *laneScratch, prefix []byte, msgs [][]byte, out [][Size]byte, lanes *[simdLanes]int, nb int) { + if cap(sp.blocks) < nb { + sp.blocks = make([]sha256Block16, nb) + } + blocks := sp.blocks[:nb] + for lane, i := range lanes { + n := len(prefix) + len(msgs[i]) + binary.BigEndian.PutUint64(sp.lens[lane][blockSize-8:], uint64(n)*8) //nolint:gosec // G115 n is a slice length + } + pfx := &sp.pfx + for b := range blocks { + start := b * blockSize + var rows [simdLanes]archsimd.Uint8x64 + for lane, i := range lanes { + msg := msgs[i] + var row archsimd.Uint8x64 + if start >= len(prefix) { + row, _ = archsimd.LoadUint8x64Part(msg[min(start-len(prefix), len(msg)):]) + } else { + m, _ := archsimd.LoadUint8x64Part(msg) + row = m.Permute(pfx.shift).And(pfx.keep).Or(pfx.bytes) + } + if p := len(prefix) + len(msg) - start; p >= 0 && p < blockSize { + row = row.Or(archsimd.LoadUint8x64Array(&pad80[p])) + } + if b == nb-1 { + row = row.Or(archsimd.LoadUint8x64Array(&sp.lens[lane])) + } + rows[lane] = row + } + transposeBlock(&rows, &blocks[b]) + } + var st [8][16]uint32 + sha256x16(blocks, &st) + for lane, i := range lanes { + d := &out[i] + for w := range 8 { + binary.BigEndian.PutUint32(d[4*w:], st[w][lane]) + } + } +} + +// prefixShift assembles the first block of prefix || msg from a raw load of +// msg: shift moves msg byte i to position i+len(prefix), keep zeroes the +// prefix positions and bytes holds the prefix itself. The prefix must be +// shorter than a block. +type prefixShift struct { + shift, keep, bytes archsimd.Uint8x64 +} + +func (p *prefixShift) init(prefix []byte) { + var shift, keep, bytes [blockSize]byte + for i := range blockSize { + if i < len(prefix) { + bytes[i] = prefix[i] + continue + } + shift[i] = byte(i - len(prefix)) //nolint:gosec // G115 0 <= i-len(prefix) < 64 + keep[i] = 0xff + } + p.shift = archsimd.LoadUint8x64Array(&shift) + p.keep = archsimd.LoadUint8x64Array(&keep) + p.bytes = archsimd.LoadUint8x64Array(&bytes) +} + +// pad80[p] is a block with the SHA-256 terminator byte at offset p. +var pad80 = func() [blockSize][blockSize]byte { + var t [blockSize][blockSize]byte + for p := range t { + t[p][p] = 0x80 + } + return t +}() + +// bswap32 reverses the bytes of every 32-bit word (VPSHUFB within 128-bit +// groups); SHA-256 words are big-endian. +var bswap32 = func() [64]int8 { + var idx [64]int8 + for i := range idx { + idx[i] = int8((i &^ 3) + (3 - i&3)) //nolint:gosec // G115 0 <= i < 64 + } + return idx +}() + +// transposeIdx holds, per stage k, the ConcatPermute index vectors that swap +// bit k of the row index with bit k of the element index: [k][0] produces the +// row with bit k clear, [k][1] the row with bit k set. Indices 16-31 select +// from the second source. +var transposeIdx = func() [4][2][16]uint32 { + var idx [4][2][16]uint32 + for k := range 4 { + bk := uint32(1) << k + for e := range uint32(16) { + if e&bk == 0 { + idx[k][0][e] = e + idx[k][1][e] = e | bk + } else { + idx[k][0][e] = 16 + (e ^ bk) + idx[k][1][e] = 16 + e + } + } + } + return idx +}() + +// bswapRows reinterprets each 64-byte row as sixteen big-endian words. +func bswapRows(rows *[simdLanes]archsimd.Uint8x64, v *[16]archsimd.Uint32x16) { + sw := archsimd.LoadInt8x64Array(&bswap32) + for i := range v { + v[i] = rows[i].PermuteOrZeroGrouped(sw).ReshapeToUint32s() + } +} + +// transposeBlock converts rows[lane] (64 message bytes of one lane) into +// blk[word][lane] big-endian words with a four-stage in-register transpose. +func transposeBlock(rows *[simdLanes]archsimd.Uint8x64, blk *sha256Block16) { + var v [16]archsimd.Uint32x16 + bswapRows(rows, &v) + for k := range 4 { + lo := archsimd.LoadUint32x16Array(&transposeIdx[k][0]) + hi := archsimd.LoadUint32x16Array(&transposeIdx[k][1]) + bk := 1 << k + for i := range 16 { + if i&bk != 0 { + continue + } + j := i | bk + x, y := v[i], v[j] + v[i] = x.ConcatPermute(y, lo) + v[j] = x.ConcatPermute(y, hi) + } + } + for w := range v { + v[w].StoreArray(&blk[w]) + } +} diff --git a/sei-tendermint/crypto/tmhash/backend_test.go b/sei-tendermint/crypto/tmhash/backend_test.go new file mode 100644 index 0000000000..1e6f8debb7 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_test.go @@ -0,0 +1,107 @@ +package tmhash + +import ( + "crypto/sha256" + "fmt" + "math/rand/v2" + "slices" + "testing" + + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" +) + +// batchSizes covers SHA-256 block boundaries with and without the one-byte +// prefix, the 65-byte Merkle inner node and a few multi-block messages. +var batchSizes = []int{0, 1, 31, 32, 54, 55, 56, 63, 64, 65, 100, 118, 119, 120, 127, 128, 129, 200, 1000, 4096, 5000} + +func referenceSum(prefix, msg []byte) [Size]byte { + return sha256.Sum256(slices.Concat(prefix, msg)) +} + +func randomMsgs(rng *rand.Rand, n, size int) [][]byte { + msgs := make([][]byte, n) + for i := range msgs { + msgs[i] = make([]byte, size) + for j := range msgs[i] { + msgs[i][j] = byte(rng.UintN(256)) + } + } + return msgs +} + +func TestBackendsAgreeWithReference(t *testing.T) { + for _, name := range availableBackendNames() { + b := availableBackends()[name] + for _, prefix := range [][]byte{nil, {0}, {1}} { + for _, size := range batchSizes { + // One partial batch, one exact multiple, one with a remainder. + for _, n := range []int{1, 15, 16, 32, 37} { + t.Run(fmt.Sprintf("%s/prefix=%d/size=%d/n=%d", name, len(prefix), size, n), func(t *testing.T) { + rng := rand.New(rand.NewPCG(uint64(size), uint64(n))) + msgs := randomMsgs(rng, n, size) + out := make([][Size]byte, n) + b.sumBatch(prefix, msgs, out) + for i, msg := range msgs { + require.Equal(t, referenceSum(prefix, msg), out[i]) + } + }) + } + } + } + } +} + +func TestBackendsAgreeOnMixedSizes(t *testing.T) { + rng := rand.New(rand.NewPCG(1, 2)) + msgs := make([][]byte, 200) + for i := range msgs { + msgs[i] = randomMsgs(rng, 1, int(rng.UintN(600)))[0] + } + for _, name := range availableBackendNames() { + out := make([][Size]byte, len(msgs)) + availableBackends()[name].sumBatch([]byte{0}, msgs, out) + for i, msg := range msgs { + require.Equal(t, referenceSum([]byte{0}, msg), out[i]) + } + } +} + +func TestSelectBackend(t *testing.T) { + require.Equal(t, "default", selectBackend("default").name) + require.Equal(t, 1, selectBackend("default").lanes) + auto := selectBackend("") + require.Equal(t, auto.name, selectBackend("unknown").name) + if simd, ok := simdBackend(); ok { + require.Equal(t, simd.name, auto.name) + require.Equal(t, simd.name, selectBackend(simd.name).name) + } else { + require.Equal(t, "default", auto.name) + } + require.True(t, slices.Contains(availableBackendNames(), ActiveBackend())) +} + +func benchmarkSumBatch(b *testing.B, size, n int) { + rng := rand.New(rand.NewPCG(3, 4)) + msgs := randomMsgs(rng, n, size) + out := make([][Size]byte, n) + prefix := []byte{0} + for _, name := range availableBackendNames() { + be := availableBackends()[name] + b.Run("backend="+name, func(b *testing.B) { + b.SetBytes(int64(n * (size + 1))) + for b.Loop() { + be.sumBatch(prefix, msgs, out) + } + }) + } +} + +// BenchmarkSumBatchInner is a Merkle inner-node level: 1024 x 64-byte +// messages behind a one-byte prefix. +func BenchmarkSumBatchInner(b *testing.B) { benchmarkSumBatch(b, 64, 1024) } + +// BenchmarkSumBatchLeaf256 is a leaf level of 1024 x 256-byte items. +func BenchmarkSumBatchLeaf256(b *testing.B) { benchmarkSumBatch(b, 256, 1024) } + +// BenchmarkSumBatchLeaf1K is a leaf level of 1024 x 1 KiB items. +func BenchmarkSumBatchLeaf1K(b *testing.B) { benchmarkSumBatch(b, 1024, 1024) } diff --git a/sei-tendermint/crypto/tmhash/gen_sha256x16.go b/sei-tendermint/crypto/tmhash/gen_sha256x16.go new file mode 100644 index 0000000000..5c0aaab396 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/gen_sha256x16.go @@ -0,0 +1,138 @@ +//go:build ignore + +// gen_sha256x16 writes sha256x16_amd64.go: a fully unrolled 16-lane SHA-256 +// compression over simd/archsimd Uint32x16 vectors. Each lane is one message; +// all lanes must have the same number of padded blocks. +// +// Usage: go run gen_sha256x16.go +package main + +import ( + "bytes" + "fmt" + "go/format" + "os" +) + +var k = [64]uint32{ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +} + +var h0 = [8]uint32{ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +} + +func main() { + var b bytes.Buffer + p := func(format string, args ...any) { fmt.Fprintf(&b, format+"\n", args...) } + + p("// Code generated by gen_sha256x16.go; DO NOT EDIT.") + p("") + p("//go:build goexperiment.simd && amd64") + p("") + p("package tmhash") + p("") + p(`import "simd/archsimd"`) + p("") + p("// sha256Block16 is one 64-byte block of each of 16 messages, laid out") + p("// block16[word][lane] so that a word of every lane loads as one vector.") + p("type sha256Block16 [16][16]uint32") + p("") + p("// Constants are kept pre-broadcast and loaded with LoadUint32x16Array;") + p("// archsimd.BroadcastUint32x16 compiles to a legacy-SSE sequence that costs") + p("// an SSE/AVX transition on every call.") + p("var (") + p("\tsha256K16 [64][16]uint32") + p("\tsha256H16 [8][16]uint32") + p(")") + p("") + p("func init() {") + p("\tfor t, v := range [64]uint32{") + for t := 0; t < 64; t += 8 { + p("\t\t0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x,", + k[t], k[t+1], k[t+2], k[t+3], k[t+4], k[t+5], k[t+6], k[t+7]) + } + p("\t} {") + p("\t\tfor lane := range sha256K16[t] {") + p("\t\t\tsha256K16[t][lane] = v") + p("\t\t}") + p("\t}") + p("\tfor i, v := range [8]uint32{") + p("\t\t0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x,", + h0[0], h0[1], h0[2], h0[3], h0[4], h0[5], h0[6], h0[7]) + p("\t} {") + p("\t\tfor lane := range sha256H16[i] {") + p("\t\t\tsha256H16[i][lane] = v") + p("\t\t}") + p("\t}") + p("}") + p("") + p("// rotr32 rotates every lane right by n using VPSHRDD; RotateAllRight is") + p("// emulated with three instructions.") + p("func rotr32(x archsimd.Uint32x16, n uint64) archsimd.Uint32x16 {") + p("\treturn x.ShiftAllRightConcatMod32(x, n)") + p("}") + p("") + p("// sha256x16 compresses blocks, one sha256Block16 per message block, and") + p("// writes the final state as out[word][lane] (big-endian digest words).") + p("func sha256x16(blocks []sha256Block16, out *[8][16]uint32) {") + for i := 0; i < 8; i++ { + p("\th%d := archsimd.LoadUint32x16Array(&sha256H16[%d])", i, i) + } + p("\tfor bi := range blocks {") + p("\t\tblk := &blocks[bi]") + for t := 0; t < 16; t++ { + p("\t\tw%d := archsimd.LoadUint32x16Array(&blk[%d])", t, t) + } + names := [8]string{"h0", "h1", "h2", "h3", "h4", "h5", "h6", "h7"} + // Working variables a..h are renamed each round instead of shuffled. + v := [8]string{"a", "b", "c", "d", "e", "f", "g", "h"} + for i := range v { + p("\t\t%s := %s", v[i], names[i]) + } + p("\t\tvar t1, t2 archsimd.Uint32x16") + for t := 0; t < 64; t++ { + if t >= 16 { + w16, w15, w7, w2 := t%16, (t-15)%16, (t-7)%16, (t-2)%16 + p("\t\tw%d = w%d.Add(rotr32(w%d, 7).Xor(rotr32(w%d, 18)).Xor(w%d.ShiftAllRight(3))).Add(w%d).Add(rotr32(w%d, 17).Xor(rotr32(w%d, 19)).Xor(w%d.ShiftAllRight(10)))", + w16, w16, w15, w15, w15, w7, w2, w2, w2) + } + a, bb, c, d, e, f, g, hh := v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7] + // T1 = h + Σ1(e) + Ch(e,f,g) + K[t] + W[t] + p("\t\tt1 = %s.Add(rotr32(%s, 6).Xor(rotr32(%s, 11)).Xor(rotr32(%s, 25))).Add(%s.And(%s).Xor(%s.AndNot(%s))).Add(archsimd.LoadUint32x16Array(&sha256K16[%d])).Add(w%d)", + hh, e, e, e, e, f, g, e, t, t%16) + // T2 = Σ0(a) + Maj(a,b,c), Maj = (a&b) ^ ((a^b)&c) + p("\t\tt2 = rotr32(%s, 2).Xor(rotr32(%s, 13)).Xor(rotr32(%s, 22)).Add(%s.And(%s).Xor(%s.Xor(%s).And(%s)))", + a, a, a, a, bb, a, bb, c) + // h <- g, g <- f, f <- e, e <- d+T1, d <- c, c <- b, b <- a, a <- T1+T2. + // Rotating the names makes old h the new a and old d the new e. + p("\t\t%s = %s.Add(t1)", d, d) + p("\t\t%s = t1.Add(t2)", hh) + v = [8]string{hh, a, bb, c, d, e, f, g} + } + for i := range v { + p("\t\t%s = %s.Add(%s)", names[i], names[i], v[i]) + } + p("\t}") + for i := 0; i < 8; i++ { + p("\th%d.StoreArray(&out[%d])", i, i) + } + p("}") + + src, err := format.Source(b.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := os.WriteFile("sha256x16_amd64.go", src, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/sei-tendermint/crypto/tmhash/sha256x16_amd64.go b/sei-tendermint/crypto/tmhash/sha256x16_amd64.go new file mode 100644 index 0000000000..3f8d1229c7 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/sha256x16_amd64.go @@ -0,0 +1,410 @@ +// Code generated by gen_sha256x16.go; DO NOT EDIT. + +//go:build goexperiment.simd && amd64 + +package tmhash + +import "simd/archsimd" + +// sha256Block16 is one 64-byte block of each of 16 messages, laid out +// block16[word][lane] so that a word of every lane loads as one vector. +type sha256Block16 [16][16]uint32 + +// Constants are kept pre-broadcast and loaded with LoadUint32x16Array; +// archsimd.BroadcastUint32x16 compiles to a legacy-SSE sequence that costs +// an SSE/AVX transition on every call. +var ( + sha256K16 [64][16]uint32 + sha256H16 [8][16]uint32 +) + +func init() { + for t, v := range [64]uint32{ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + } { + for lane := range sha256K16[t] { + sha256K16[t][lane] = v + } + } + for i, v := range [8]uint32{ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + } { + for lane := range sha256H16[i] { + sha256H16[i][lane] = v + } + } +} + +// rotr32 rotates every lane right by n using VPSHRDD; RotateAllRight is +// emulated with three instructions. +func rotr32(x archsimd.Uint32x16, n uint64) archsimd.Uint32x16 { + return x.ShiftAllRightConcatMod32(x, n) +} + +// sha256x16 compresses blocks, one sha256Block16 per message block, and +// writes the final state as out[word][lane] (big-endian digest words). +func sha256x16(blocks []sha256Block16, out *[8][16]uint32) { + h0 := archsimd.LoadUint32x16Array(&sha256H16[0]) + h1 := archsimd.LoadUint32x16Array(&sha256H16[1]) + h2 := archsimd.LoadUint32x16Array(&sha256H16[2]) + h3 := archsimd.LoadUint32x16Array(&sha256H16[3]) + h4 := archsimd.LoadUint32x16Array(&sha256H16[4]) + h5 := archsimd.LoadUint32x16Array(&sha256H16[5]) + h6 := archsimd.LoadUint32x16Array(&sha256H16[6]) + h7 := archsimd.LoadUint32x16Array(&sha256H16[7]) + for bi := range blocks { + blk := &blocks[bi] + w0 := archsimd.LoadUint32x16Array(&blk[0]) + w1 := archsimd.LoadUint32x16Array(&blk[1]) + w2 := archsimd.LoadUint32x16Array(&blk[2]) + w3 := archsimd.LoadUint32x16Array(&blk[3]) + w4 := archsimd.LoadUint32x16Array(&blk[4]) + w5 := archsimd.LoadUint32x16Array(&blk[5]) + w6 := archsimd.LoadUint32x16Array(&blk[6]) + w7 := archsimd.LoadUint32x16Array(&blk[7]) + w8 := archsimd.LoadUint32x16Array(&blk[8]) + w9 := archsimd.LoadUint32x16Array(&blk[9]) + w10 := archsimd.LoadUint32x16Array(&blk[10]) + w11 := archsimd.LoadUint32x16Array(&blk[11]) + w12 := archsimd.LoadUint32x16Array(&blk[12]) + w13 := archsimd.LoadUint32x16Array(&blk[13]) + w14 := archsimd.LoadUint32x16Array(&blk[14]) + w15 := archsimd.LoadUint32x16Array(&blk[15]) + a := h0 + b := h1 + c := h2 + d := h3 + e := h4 + f := h5 + g := h6 + h := h7 + var t1, t2 archsimd.Uint32x16 + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[0])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[1])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[2])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[3])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[4])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[5])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[6])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[7])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[8])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[9])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[10])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[11])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[12])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[13])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[14])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[15])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w0 = w0.Add(rotr32(w1, 7).Xor(rotr32(w1, 18)).Xor(w1.ShiftAllRight(3))).Add(w9).Add(rotr32(w14, 17).Xor(rotr32(w14, 19)).Xor(w14.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[16])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w1 = w1.Add(rotr32(w2, 7).Xor(rotr32(w2, 18)).Xor(w2.ShiftAllRight(3))).Add(w10).Add(rotr32(w15, 17).Xor(rotr32(w15, 19)).Xor(w15.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[17])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w2 = w2.Add(rotr32(w3, 7).Xor(rotr32(w3, 18)).Xor(w3.ShiftAllRight(3))).Add(w11).Add(rotr32(w0, 17).Xor(rotr32(w0, 19)).Xor(w0.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[18])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w3 = w3.Add(rotr32(w4, 7).Xor(rotr32(w4, 18)).Xor(w4.ShiftAllRight(3))).Add(w12).Add(rotr32(w1, 17).Xor(rotr32(w1, 19)).Xor(w1.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[19])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w4 = w4.Add(rotr32(w5, 7).Xor(rotr32(w5, 18)).Xor(w5.ShiftAllRight(3))).Add(w13).Add(rotr32(w2, 17).Xor(rotr32(w2, 19)).Xor(w2.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[20])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w5 = w5.Add(rotr32(w6, 7).Xor(rotr32(w6, 18)).Xor(w6.ShiftAllRight(3))).Add(w14).Add(rotr32(w3, 17).Xor(rotr32(w3, 19)).Xor(w3.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[21])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w6 = w6.Add(rotr32(w7, 7).Xor(rotr32(w7, 18)).Xor(w7.ShiftAllRight(3))).Add(w15).Add(rotr32(w4, 17).Xor(rotr32(w4, 19)).Xor(w4.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[22])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w7 = w7.Add(rotr32(w8, 7).Xor(rotr32(w8, 18)).Xor(w8.ShiftAllRight(3))).Add(w0).Add(rotr32(w5, 17).Xor(rotr32(w5, 19)).Xor(w5.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[23])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w8 = w8.Add(rotr32(w9, 7).Xor(rotr32(w9, 18)).Xor(w9.ShiftAllRight(3))).Add(w1).Add(rotr32(w6, 17).Xor(rotr32(w6, 19)).Xor(w6.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[24])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w9 = w9.Add(rotr32(w10, 7).Xor(rotr32(w10, 18)).Xor(w10.ShiftAllRight(3))).Add(w2).Add(rotr32(w7, 17).Xor(rotr32(w7, 19)).Xor(w7.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[25])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w10 = w10.Add(rotr32(w11, 7).Xor(rotr32(w11, 18)).Xor(w11.ShiftAllRight(3))).Add(w3).Add(rotr32(w8, 17).Xor(rotr32(w8, 19)).Xor(w8.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[26])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w11 = w11.Add(rotr32(w12, 7).Xor(rotr32(w12, 18)).Xor(w12.ShiftAllRight(3))).Add(w4).Add(rotr32(w9, 17).Xor(rotr32(w9, 19)).Xor(w9.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[27])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w12 = w12.Add(rotr32(w13, 7).Xor(rotr32(w13, 18)).Xor(w13.ShiftAllRight(3))).Add(w5).Add(rotr32(w10, 17).Xor(rotr32(w10, 19)).Xor(w10.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[28])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w13 = w13.Add(rotr32(w14, 7).Xor(rotr32(w14, 18)).Xor(w14.ShiftAllRight(3))).Add(w6).Add(rotr32(w11, 17).Xor(rotr32(w11, 19)).Xor(w11.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[29])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w14 = w14.Add(rotr32(w15, 7).Xor(rotr32(w15, 18)).Xor(w15.ShiftAllRight(3))).Add(w7).Add(rotr32(w12, 17).Xor(rotr32(w12, 19)).Xor(w12.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[30])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w15 = w15.Add(rotr32(w0, 7).Xor(rotr32(w0, 18)).Xor(w0.ShiftAllRight(3))).Add(w8).Add(rotr32(w13, 17).Xor(rotr32(w13, 19)).Xor(w13.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[31])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w0 = w0.Add(rotr32(w1, 7).Xor(rotr32(w1, 18)).Xor(w1.ShiftAllRight(3))).Add(w9).Add(rotr32(w14, 17).Xor(rotr32(w14, 19)).Xor(w14.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[32])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w1 = w1.Add(rotr32(w2, 7).Xor(rotr32(w2, 18)).Xor(w2.ShiftAllRight(3))).Add(w10).Add(rotr32(w15, 17).Xor(rotr32(w15, 19)).Xor(w15.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[33])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w2 = w2.Add(rotr32(w3, 7).Xor(rotr32(w3, 18)).Xor(w3.ShiftAllRight(3))).Add(w11).Add(rotr32(w0, 17).Xor(rotr32(w0, 19)).Xor(w0.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[34])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w3 = w3.Add(rotr32(w4, 7).Xor(rotr32(w4, 18)).Xor(w4.ShiftAllRight(3))).Add(w12).Add(rotr32(w1, 17).Xor(rotr32(w1, 19)).Xor(w1.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[35])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w4 = w4.Add(rotr32(w5, 7).Xor(rotr32(w5, 18)).Xor(w5.ShiftAllRight(3))).Add(w13).Add(rotr32(w2, 17).Xor(rotr32(w2, 19)).Xor(w2.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[36])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w5 = w5.Add(rotr32(w6, 7).Xor(rotr32(w6, 18)).Xor(w6.ShiftAllRight(3))).Add(w14).Add(rotr32(w3, 17).Xor(rotr32(w3, 19)).Xor(w3.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[37])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w6 = w6.Add(rotr32(w7, 7).Xor(rotr32(w7, 18)).Xor(w7.ShiftAllRight(3))).Add(w15).Add(rotr32(w4, 17).Xor(rotr32(w4, 19)).Xor(w4.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[38])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w7 = w7.Add(rotr32(w8, 7).Xor(rotr32(w8, 18)).Xor(w8.ShiftAllRight(3))).Add(w0).Add(rotr32(w5, 17).Xor(rotr32(w5, 19)).Xor(w5.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[39])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w8 = w8.Add(rotr32(w9, 7).Xor(rotr32(w9, 18)).Xor(w9.ShiftAllRight(3))).Add(w1).Add(rotr32(w6, 17).Xor(rotr32(w6, 19)).Xor(w6.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[40])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w9 = w9.Add(rotr32(w10, 7).Xor(rotr32(w10, 18)).Xor(w10.ShiftAllRight(3))).Add(w2).Add(rotr32(w7, 17).Xor(rotr32(w7, 19)).Xor(w7.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[41])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w10 = w10.Add(rotr32(w11, 7).Xor(rotr32(w11, 18)).Xor(w11.ShiftAllRight(3))).Add(w3).Add(rotr32(w8, 17).Xor(rotr32(w8, 19)).Xor(w8.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[42])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w11 = w11.Add(rotr32(w12, 7).Xor(rotr32(w12, 18)).Xor(w12.ShiftAllRight(3))).Add(w4).Add(rotr32(w9, 17).Xor(rotr32(w9, 19)).Xor(w9.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[43])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w12 = w12.Add(rotr32(w13, 7).Xor(rotr32(w13, 18)).Xor(w13.ShiftAllRight(3))).Add(w5).Add(rotr32(w10, 17).Xor(rotr32(w10, 19)).Xor(w10.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[44])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w13 = w13.Add(rotr32(w14, 7).Xor(rotr32(w14, 18)).Xor(w14.ShiftAllRight(3))).Add(w6).Add(rotr32(w11, 17).Xor(rotr32(w11, 19)).Xor(w11.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[45])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w14 = w14.Add(rotr32(w15, 7).Xor(rotr32(w15, 18)).Xor(w15.ShiftAllRight(3))).Add(w7).Add(rotr32(w12, 17).Xor(rotr32(w12, 19)).Xor(w12.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[46])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w15 = w15.Add(rotr32(w0, 7).Xor(rotr32(w0, 18)).Xor(w0.ShiftAllRight(3))).Add(w8).Add(rotr32(w13, 17).Xor(rotr32(w13, 19)).Xor(w13.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[47])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w0 = w0.Add(rotr32(w1, 7).Xor(rotr32(w1, 18)).Xor(w1.ShiftAllRight(3))).Add(w9).Add(rotr32(w14, 17).Xor(rotr32(w14, 19)).Xor(w14.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[48])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w1 = w1.Add(rotr32(w2, 7).Xor(rotr32(w2, 18)).Xor(w2.ShiftAllRight(3))).Add(w10).Add(rotr32(w15, 17).Xor(rotr32(w15, 19)).Xor(w15.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[49])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w2 = w2.Add(rotr32(w3, 7).Xor(rotr32(w3, 18)).Xor(w3.ShiftAllRight(3))).Add(w11).Add(rotr32(w0, 17).Xor(rotr32(w0, 19)).Xor(w0.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[50])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w3 = w3.Add(rotr32(w4, 7).Xor(rotr32(w4, 18)).Xor(w4.ShiftAllRight(3))).Add(w12).Add(rotr32(w1, 17).Xor(rotr32(w1, 19)).Xor(w1.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[51])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w4 = w4.Add(rotr32(w5, 7).Xor(rotr32(w5, 18)).Xor(w5.ShiftAllRight(3))).Add(w13).Add(rotr32(w2, 17).Xor(rotr32(w2, 19)).Xor(w2.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[52])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w5 = w5.Add(rotr32(w6, 7).Xor(rotr32(w6, 18)).Xor(w6.ShiftAllRight(3))).Add(w14).Add(rotr32(w3, 17).Xor(rotr32(w3, 19)).Xor(w3.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[53])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w6 = w6.Add(rotr32(w7, 7).Xor(rotr32(w7, 18)).Xor(w7.ShiftAllRight(3))).Add(w15).Add(rotr32(w4, 17).Xor(rotr32(w4, 19)).Xor(w4.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[54])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w7 = w7.Add(rotr32(w8, 7).Xor(rotr32(w8, 18)).Xor(w8.ShiftAllRight(3))).Add(w0).Add(rotr32(w5, 17).Xor(rotr32(w5, 19)).Xor(w5.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[55])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w8 = w8.Add(rotr32(w9, 7).Xor(rotr32(w9, 18)).Xor(w9.ShiftAllRight(3))).Add(w1).Add(rotr32(w6, 17).Xor(rotr32(w6, 19)).Xor(w6.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[56])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w9 = w9.Add(rotr32(w10, 7).Xor(rotr32(w10, 18)).Xor(w10.ShiftAllRight(3))).Add(w2).Add(rotr32(w7, 17).Xor(rotr32(w7, 19)).Xor(w7.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[57])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w10 = w10.Add(rotr32(w11, 7).Xor(rotr32(w11, 18)).Xor(w11.ShiftAllRight(3))).Add(w3).Add(rotr32(w8, 17).Xor(rotr32(w8, 19)).Xor(w8.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[58])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w11 = w11.Add(rotr32(w12, 7).Xor(rotr32(w12, 18)).Xor(w12.ShiftAllRight(3))).Add(w4).Add(rotr32(w9, 17).Xor(rotr32(w9, 19)).Xor(w9.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[59])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w12 = w12.Add(rotr32(w13, 7).Xor(rotr32(w13, 18)).Xor(w13.ShiftAllRight(3))).Add(w5).Add(rotr32(w10, 17).Xor(rotr32(w10, 19)).Xor(w10.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[60])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w13 = w13.Add(rotr32(w14, 7).Xor(rotr32(w14, 18)).Xor(w14.ShiftAllRight(3))).Add(w6).Add(rotr32(w11, 17).Xor(rotr32(w11, 19)).Xor(w11.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[61])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w14 = w14.Add(rotr32(w15, 7).Xor(rotr32(w15, 18)).Xor(w15.ShiftAllRight(3))).Add(w7).Add(rotr32(w12, 17).Xor(rotr32(w12, 19)).Xor(w12.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[62])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w15 = w15.Add(rotr32(w0, 7).Xor(rotr32(w0, 18)).Xor(w0.ShiftAllRight(3))).Add(w8).Add(rotr32(w13, 17).Xor(rotr32(w13, 19)).Xor(w13.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[63])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + h0 = h0.Add(a) + h1 = h1.Add(b) + h2 = h2.Add(c) + h3 = h3.Add(d) + h4 = h4.Add(e) + h5 = h5.Add(f) + h6 = h6.Add(g) + h7 = h7.Add(h) + } + h0.StoreArray(&out[0]) + h1.StoreArray(&out[1]) + h2.StoreArray(&out[2]) + h3.StoreArray(&out[3]) + h4.StoreArray(&out[4]) + h5.StoreArray(&out[5]) + h6.StoreArray(&out[6]) + h7.StoreArray(&out[7]) +} diff --git a/sei-tendermint/crypto/tmhash/zeroupper_amd64.s b/sei-tendermint/crypto/tmhash/zeroupper_amd64.s new file mode 100644 index 0000000000..6c4dd6d709 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/zeroupper_amd64.s @@ -0,0 +1,8 @@ +//go:build goexperiment.simd && amd64 + +#include "textflag.h" + +// func vzeroupper() +TEXT ·vzeroupper(SB), NOSPLIT, $0-0 + VZEROUPPER + RET From ae086561ff188aef14db577ca7aa7b9be9a6283e Mon Sep 17 00:00:00 2001 From: masih Date: Mon, 14 Sep 2026 16:54:07 +0000 Subject: [PATCH 5/9] Tidy tmhash scalar fallback and widen batched Merkle differential test --- sei-tendermint/crypto/merkle/tree_test.go | 4 +-- sei-tendermint/crypto/tmhash/backend.go | 16 +---------- .../crypto/tmhash/backend_default.go | 28 +++++++++++++++---- .../crypto/tmhash/backend_simd_amd64.go | 23 +++++---------- sei-tendermint/crypto/tmhash/backend_test.go | 5 ++++ 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/sei-tendermint/crypto/merkle/tree_test.go b/sei-tendermint/crypto/merkle/tree_test.go index 2801967598..7280f99bc5 100644 --- a/sei-tendermint/crypto/merkle/tree_test.go +++ b/sei-tendermint/crypto/merkle/tree_test.go @@ -161,8 +161,8 @@ func BenchmarkHashAlternatives(b *testing.B) { // recursive reference around every lane-count boundary. func TestHashFromByteSlicesBatched(t *testing.T) { sha := sha256.New() - for _, size := range []int{2, 32, 128} { - for total := 1; total <= 70; total++ { + for _, size := range []int{0, 2, 32, 128} { + for total := 1; total <= 130; total++ { items := make([][]byte, total) for i := range items { items[i] = tmrand.Bytes(size) diff --git a/sei-tendermint/crypto/tmhash/backend.go b/sei-tendermint/crypto/tmhash/backend.go index 0e70c415a2..04dc097c5f 100644 --- a/sei-tendermint/crypto/tmhash/backend.go +++ b/sei-tendermint/crypto/tmhash/backend.go @@ -1,9 +1,6 @@ package tmhash -import ( - "os" - "slices" -) +import "os" // BackendEnv is the environment variable that pins the batch hashing backend // by name. An unknown or empty value leaves the selection automatic. @@ -49,17 +46,6 @@ func availableBackends() map[string]backend { return m } -// availableBackendNames returns the names from availableBackends, sorted. -func availableBackendNames() []string { - m := availableBackends() - names := make([]string, 0, len(m)) - for name := range m { - names = append(names, name) - } - slices.Sort(names) - return names -} - // selectBackend picks the backend named by pin, or the fastest available one // when pin is empty or unknown. func selectBackend(pin string) backend { diff --git a/sei-tendermint/crypto/tmhash/backend_default.go b/sei-tendermint/crypto/tmhash/backend_default.go index 33767ed6dd..8211c23edd 100644 --- a/sei-tendermint/crypto/tmhash/backend_default.go +++ b/sei-tendermint/crypto/tmhash/backend_default.go @@ -1,6 +1,9 @@ package tmhash -import "crypto/sha256" +import ( + "crypto/sha256" + "hash" +) // defaultBackend hashes one message at a time with crypto/sha256, which uses // the SHA-NI single-lane instructions where the CPU has them. @@ -13,9 +16,24 @@ var defaultBackend = backend{ func sumBatchScalar(prefix []byte, msgs [][]byte, out [][Size]byte) { h := sha256.New() for i, msg := range msgs { - h.Reset() - h.Write(prefix) - h.Write(msg) - h.Sum(out[i][:0]) + sumOne(h, prefix, msg, &out[i]) } } + +// sumScalarAt hashes msgs[i] into out[i] for every i in idx. +func sumScalarAt(prefix []byte, msgs [][]byte, out [][Size]byte, idx []int) { + if len(idx) == 0 { + return + } + h := sha256.New() + for _, i := range idx { + sumOne(h, prefix, msgs[i], &out[i]) + } +} + +func sumOne(h hash.Hash, prefix, msg []byte, out *[Size]byte) { + h.Reset() + h.Write(prefix) + h.Write(msg) + h.Sum(out[:0]) +} diff --git a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go index d715438484..883f169ed1 100644 --- a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go +++ b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go @@ -3,7 +3,6 @@ package tmhash import ( - "crypto/sha256" "encoding/binary" "sync" @@ -12,9 +11,7 @@ import ( //go:generate go run gen_sha256x16.go -// vzeroupper clears the upper halves of the vector registers. The compiler -// does not emit it after AVX-512 code, and legacy-SSE code that follows (the -// SHA-NI scalar path, memmove) runs several times slower while they are dirty. +// vzeroupper clears the upper halves of the vector registers. // //go:noescape func vzeroupper() @@ -68,9 +65,8 @@ func paddedBlocks(n int) int { return (n + 1 + 8 + blockSize - 1) / blockSize } -// sumBatchSIMD hashes msgs sixteen at a time. Lanes must share a block count, -// so messages are bucketed by padded length; buckets with fewer than sixteen -// messages left over fall back to the scalar backend. +// sumBatchSIMD hashes sixteen messages of equal padded block count at a time +// and hands whatever does not fill a group of sixteen to the scalar backend. func sumBatchSIMD(prefix []byte, msgs [][]byte, out [][Size]byte) { if len(msgs) < simdLanes || len(prefix) >= blockSize { sumBatchScalar(prefix, msgs, out) @@ -115,16 +111,11 @@ func sumBatchSIMD(prefix []byte, msgs [][]byte, out [][Size]byte) { rest = append(rest, idx...) } laneScratchPool.Put(sp) + // The compiler emits no VZEROUPPER after AVX-512 code, and the legacy-SSE + // SHA-NI path below runs several times slower while the upper halves are + // dirty. vzeroupper() - if len(rest) > 0 { - h := sha256.New() - for _, i := range rest { - h.Reset() - h.Write(prefix) - h.Write(msgs[i]) - h.Sum(out[i][:0]) - } - } + sumScalarAt(prefix, msgs, out, rest) } // sha256Lanes hashes the sixteen messages selected by lanes, each of nb diff --git a/sei-tendermint/crypto/tmhash/backend_test.go b/sei-tendermint/crypto/tmhash/backend_test.go index 1e6f8debb7..9b2cd7a321 100644 --- a/sei-tendermint/crypto/tmhash/backend_test.go +++ b/sei-tendermint/crypto/tmhash/backend_test.go @@ -3,6 +3,7 @@ package tmhash import ( "crypto/sha256" "fmt" + "maps" "math/rand/v2" "slices" "testing" @@ -14,6 +15,10 @@ import ( // prefix, the 65-byte Merkle inner node and a few multi-block messages. var batchSizes = []int{0, 1, 31, 32, 54, 55, 56, 63, 64, 65, 100, 118, 119, 120, 127, 128, 129, 200, 1000, 4096, 5000} +func availableBackendNames() []string { + return slices.Sorted(maps.Keys(availableBackends())) +} + func referenceSum(prefix, msg []byte) [Size]byte { return sha256.Sum256(slices.Concat(prefix, msg)) } From 2a05ac80c7abb9e870813b30db612b99c30ebf48 Mon Sep 17 00:00:00 2001 From: masih Date: Tue, 15 Sep 2026 08:02:35 +0000 Subject: [PATCH 6/9] Use utils.TestRng in tmhash backend tests and range-over-int in the generator --- sei-tendermint/crypto/tmhash/backend_test.go | 18 +++++++----------- sei-tendermint/crypto/tmhash/gen_sha256x16.go | 8 ++++---- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/sei-tendermint/crypto/tmhash/backend_test.go b/sei-tendermint/crypto/tmhash/backend_test.go index 9b2cd7a321..cba23f38c5 100644 --- a/sei-tendermint/crypto/tmhash/backend_test.go +++ b/sei-tendermint/crypto/tmhash/backend_test.go @@ -4,10 +4,10 @@ import ( "crypto/sha256" "fmt" "maps" - "math/rand/v2" "slices" "testing" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" ) @@ -23,13 +23,10 @@ func referenceSum(prefix, msg []byte) [Size]byte { return sha256.Sum256(slices.Concat(prefix, msg)) } -func randomMsgs(rng *rand.Rand, n, size int) [][]byte { +func randomMsgs(rng utils.Rng, n, size int) [][]byte { msgs := make([][]byte, n) for i := range msgs { - msgs[i] = make([]byte, size) - for j := range msgs[i] { - msgs[i][j] = byte(rng.UintN(256)) - } + msgs[i] = utils.GenBytes(rng, size) } return msgs } @@ -42,7 +39,7 @@ func TestBackendsAgreeWithReference(t *testing.T) { // One partial batch, one exact multiple, one with a remainder. for _, n := range []int{1, 15, 16, 32, 37} { t.Run(fmt.Sprintf("%s/prefix=%d/size=%d/n=%d", name, len(prefix), size, n), func(t *testing.T) { - rng := rand.New(rand.NewPCG(uint64(size), uint64(n))) + rng := utils.TestRng() msgs := randomMsgs(rng, n, size) out := make([][Size]byte, n) b.sumBatch(prefix, msgs, out) @@ -57,10 +54,10 @@ func TestBackendsAgreeWithReference(t *testing.T) { } func TestBackendsAgreeOnMixedSizes(t *testing.T) { - rng := rand.New(rand.NewPCG(1, 2)) + rng := utils.TestRng() msgs := make([][]byte, 200) for i := range msgs { - msgs[i] = randomMsgs(rng, 1, int(rng.UintN(600)))[0] + msgs[i] = utils.GenBytes(rng, rng.Intn(600)) } for _, name := range availableBackendNames() { out := make([][Size]byte, len(msgs)) @@ -86,8 +83,7 @@ func TestSelectBackend(t *testing.T) { } func benchmarkSumBatch(b *testing.B, size, n int) { - rng := rand.New(rand.NewPCG(3, 4)) - msgs := randomMsgs(rng, n, size) + msgs := randomMsgs(utils.TestRng(), n, size) out := make([][Size]byte, n) prefix := []byte{0} for _, name := range availableBackendNames() { diff --git a/sei-tendermint/crypto/tmhash/gen_sha256x16.go b/sei-tendermint/crypto/tmhash/gen_sha256x16.go index 5c0aaab396..308205c3c6 100644 --- a/sei-tendermint/crypto/tmhash/gen_sha256x16.go +++ b/sei-tendermint/crypto/tmhash/gen_sha256x16.go @@ -83,12 +83,12 @@ func main() { p("// sha256x16 compresses blocks, one sha256Block16 per message block, and") p("// writes the final state as out[word][lane] (big-endian digest words).") p("func sha256x16(blocks []sha256Block16, out *[8][16]uint32) {") - for i := 0; i < 8; i++ { + for i := range 8 { p("\th%d := archsimd.LoadUint32x16Array(&sha256H16[%d])", i, i) } p("\tfor bi := range blocks {") p("\t\tblk := &blocks[bi]") - for t := 0; t < 16; t++ { + for t := range 16 { p("\t\tw%d := archsimd.LoadUint32x16Array(&blk[%d])", t, t) } names := [8]string{"h0", "h1", "h2", "h3", "h4", "h5", "h6", "h7"} @@ -98,7 +98,7 @@ func main() { p("\t\t%s := %s", v[i], names[i]) } p("\t\tvar t1, t2 archsimd.Uint32x16") - for t := 0; t < 64; t++ { + for t := range 64 { if t >= 16 { w16, w15, w7, w2 := t%16, (t-15)%16, (t-7)%16, (t-2)%16 p("\t\tw%d = w%d.Add(rotr32(w%d, 7).Xor(rotr32(w%d, 18)).Xor(w%d.ShiftAllRight(3))).Add(w%d).Add(rotr32(w%d, 17).Xor(rotr32(w%d, 19)).Xor(w%d.ShiftAllRight(10)))", @@ -121,7 +121,7 @@ func main() { p("\t\t%s = %s.Add(%s)", names[i], names[i], v[i]) } p("\t}") - for i := 0; i < 8; i++ { + for i := range 8 { p("\th%d.StoreArray(&out[%d])", i, i) } p("}") From 3da844a3d4d95a65c972592e92d1e5aa1c602001 Mon Sep 17 00:00:00 2001 From: masih Date: Tue, 15 Sep 2026 08:14:01 +0000 Subject: [PATCH 7/9] Benchmark only the active tmhash backend and cover oversized messages in mixed batches --- sei-tendermint/crypto/tmhash/backend_test.go | 65 +++++++++++++++----- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/sei-tendermint/crypto/tmhash/backend_test.go b/sei-tendermint/crypto/tmhash/backend_test.go index cba23f38c5..c75112e67d 100644 --- a/sei-tendermint/crypto/tmhash/backend_test.go +++ b/sei-tendermint/crypto/tmhash/backend_test.go @@ -53,19 +53,52 @@ func TestBackendsAgreeWithReference(t *testing.T) { } } +// checkBatch hashes msgs with every available backend and compares each +// digest with crypto/sha256. +func checkBatch(t testing.TB, prefix []byte, msgs [][]byte) { + for _, name := range availableBackendNames() { + out := make([][Size]byte, len(msgs)) + availableBackends()[name].sumBatch(prefix, msgs, out) + for i, msg := range msgs { + require.Equal(t, referenceSum(prefix, msg), out[i], "%s msg %d len %d", name, i, len(msg)) + } + } +} + +// TestBackendsAgreeOnMixedSizes mixes, in one call, sizes that fill SIMD +// lanes, sizes that land in the smaller buckets and a few messages beyond +// the SIMD kernel's block limit. func TestBackendsAgreeOnMixedSizes(t *testing.T) { rng := utils.TestRng() msgs := make([][]byte, 200) for i := range msgs { - msgs[i] = utils.GenBytes(rng, rng.Intn(600)) - } - for _, name := range availableBackendNames() { - out := make([][Size]byte, len(msgs)) - availableBackends()[name].sumBatch([]byte{0}, msgs, out) - for i, msg := range msgs { - require.Equal(t, referenceSum([]byte{0}, msg), out[i]) + size := rng.Intn(600) + if rng.Intn(20) == 0 { + size = 4096 + rng.Intn(2000) } + msgs[i] = utils.GenBytes(rng, size) } + checkBatch(t, []byte{0}, msgs) +} + +// FuzzSumBatch drives every backend with an arbitrary prefix and message +// length list against crypto/sha256. Each byte of lens is one message whose +// length is the byte value scaled by 24, so that lengths span from 0 to +// beyond the SIMD kernel's block limit. +func FuzzSumBatch(f *testing.F) { + f.Add([]byte{0}, []byte{2, 3, 200, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}) + f.Add([]byte{}, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + f.Fuzz(func(t *testing.T, prefix, lens []byte) { + if len(prefix) > 63 { + prefix = prefix[:63] + } + rng := utils.TestRng() + msgs := make([][]byte, len(lens)) + for i, l := range lens { + msgs[i] = utils.GenBytes(rng, int(l)*24) + } + checkBatch(t, prefix, msgs) + }) } func TestSelectBackend(t *testing.T) { @@ -82,19 +115,19 @@ func TestSelectBackend(t *testing.T) { require.True(t, slices.Contains(availableBackendNames(), ActiveBackend())) } +// benchmarkSumBatch measures the active backend only, named after it so runs +// under different SEI_TMHASH_BACKEND values or builds compare with benchstat +// without one build's scalar samples polluting the other's column. func benchmarkSumBatch(b *testing.B, size, n int) { msgs := randomMsgs(utils.TestRng(), n, size) out := make([][Size]byte, n) prefix := []byte{0} - for _, name := range availableBackendNames() { - be := availableBackends()[name] - b.Run("backend="+name, func(b *testing.B) { - b.SetBytes(int64(n * (size + 1))) - for b.Loop() { - be.sumBatch(prefix, msgs, out) - } - }) - } + b.Run("backend="+ActiveBackend(), func(b *testing.B) { + b.SetBytes(int64(n * (size + 1))) + for b.Loop() { + SumBatch(prefix, msgs, out) + } + }) } // BenchmarkSumBatchInner is a Merkle inner-node level: 1024 x 64-byte From 5e2042f915616cec994c61e6d9295ebc3ab685de Mon Sep 17 00:00:00 2001 From: masih Date: Wed, 16 Sep 2026 14:25:24 +0000 Subject: [PATCH 8/9] Rename the SIMD benchmark workflow, sort its paths, and document simdLanes --- .../{lthash-bench.yml => simd-hash-bench.yml} | 10 +++++----- sei-tendermint/crypto/tmhash/backend_simd_amd64.go | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) rename .github/workflows/{lthash-bench.yml => simd-hash-bench.yml} (98%) diff --git a/.github/workflows/lthash-bench.yml b/.github/workflows/simd-hash-bench.yml similarity index 98% rename from .github/workflows/lthash-bench.yml rename to .github/workflows/simd-hash-bench.yml index 03b7b78274..46be8a749c 100644 --- a/.github/workflows/lthash-bench.yml +++ b/.github/workflows/simd-hash-bench.yml @@ -4,16 +4,16 @@ on: pull_request: paths: - 'sei-db/state_db/sc/flatkv/lthash/**' - - 'sei-tendermint/crypto/tmhash/**' - 'sei-tendermint/crypto/merkle/**' - - '.github/workflows/lthash-bench.yml' + - 'sei-tendermint/crypto/tmhash/**' + - '.github/workflows/simd-hash-bench.yml' push: branches: - main paths: - 'sei-db/state_db/sc/flatkv/lthash/**' - - 'sei-tendermint/crypto/tmhash/**' - 'sei-tendermint/crypto/merkle/**' + - 'sei-tendermint/crypto/tmhash/**' concurrency: cancel-in-progress: true @@ -107,7 +107,7 @@ jobs: continue-on-error: true uses: actions/github-script@v8 env: - MARKER: '' + MARKER: '' with: script: | const fs = require('fs'); @@ -125,7 +125,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: lthash-bench-${{ matrix.runner }} + name: simd-hash-bench-${{ matrix.runner }} path: | bench.txt benchstat.md diff --git a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go index 883f169ed1..620fa0445a 100644 --- a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go +++ b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go @@ -18,8 +18,10 @@ func vzeroupper() const ( simdBackendName = "simd" - simdLanes = 16 - blockSize = 64 + // simdLanes is the number of 32-bit SHA-256 words in one 512-bit ZMM + // register, i.e. how many independent messages one kernel pass hashes. + simdLanes = 16 + blockSize = 64 // simdMaxBlocks bounds the per-lane message length hashed by the SIMD // kernel; longer messages go through the scalar path so the block // scratch stays small. From 3e42a4de0c71302fad10ef0003a23fd4f1f60d44 Mon Sep 17 00:00:00 2001 From: masih Date: Wed, 16 Sep 2026 14:38:05 +0000 Subject: [PATCH 9/9] Document the hash byte format and lane layout in tmhash and merkle godocs --- sei-tendermint/crypto/merkle/tree.go | 4 +++- sei-tendermint/crypto/tmhash/backend.go | 5 +++-- sei-tendermint/crypto/tmhash/backend_simd_amd64.go | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sei-tendermint/crypto/merkle/tree.go b/sei-tendermint/crypto/merkle/tree.go index 662be86fd3..5b4b75f76a 100644 --- a/sei-tendermint/crypto/merkle/tree.go +++ b/sei-tendermint/crypto/merkle/tree.go @@ -9,7 +9,9 @@ import ( ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, -// in the provided order. It follows RFC-6962. +// in the provided order. It follows RFC-6962: a leaf hashes as +// SHA-256(0x00 || item), an inner node as SHA-256(0x01 || left || right), and +// the root is the same 32 bytes whichever tmhash backend is active. func HashFromByteSlices(items [][]byte) []byte { if lanes := tmhash.BatchLanes(); lanes > 1 && len(items) >= lanes { return hashFromByteSlicesBatched(items) diff --git a/sei-tendermint/crypto/tmhash/backend.go b/sei-tendermint/crypto/tmhash/backend.go index 04dc097c5f..339c6823f7 100644 --- a/sei-tendermint/crypto/tmhash/backend.go +++ b/sei-tendermint/crypto/tmhash/backend.go @@ -30,8 +30,9 @@ func BatchLanes() int { return active.lanes } -// SumBatch writes SHA-256(prefix || msgs[i]) to out[i] for every i. -// out must be at least as long as msgs. +// SumBatch writes SHA-256(prefix || msgs[i]) to out[i] for every i, as the +// 32-byte big-endian digest crypto/sha256 would produce for the same bytes, +// regardless of the active backend. out must be at least as long as msgs. func SumBatch(prefix []byte, msgs [][]byte, out [][Size]byte) { active.sumBatch(prefix, msgs, out) } diff --git a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go index 620fa0445a..88db55ff18 100644 --- a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go +++ b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go @@ -121,7 +121,10 @@ func sumBatchSIMD(prefix []byte, msgs [][]byte, out [][Size]byte) { } // sha256Lanes hashes the sixteen messages selected by lanes, each of nb -// padded blocks, with one kernel call. +// padded blocks, with one kernel call. Each lane's blocks are the standard +// SHA-256 message layout, prefix || msg || 0x80 || zeros || 64-bit big-endian +// bit length, transposed into sha256Block16 word-major form; the state words +// are read back per lane and stored big-endian as the digest. func sha256Lanes(sp *laneScratch, prefix []byte, msgs [][]byte, out [][Size]byte, lanes *[simdLanes]int, nb int) { if cap(sp.blocks) < nb { sp.blocks = make([]sha256Block16, nb)