diff --git a/.github/workflows/lthash-bench.yml b/.github/workflows/lthash-bench.yml deleted file mode 100644 index df1bc278bd..0000000000 --- a/.github/workflows/lthash-bench.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: LtHash backends -on: - workflow_dispatch: - pull_request: - paths: - - 'sei-db/state_db/sc/flatkv/lthash/**' - - '.github/workflows/lthash-bench.yml' - push: - branches: - - main - paths: - - 'sei-db/state_db/sc/flatkv/lthash/**' - -concurrency: - cancel-in-progress: true - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} - -permissions: - contents: read - pull-requests: write - -env: - GO_VERSION: '1.27.1' - LTHASH_PKG: ./sei-db/state_db/sc/flatkv/lthash - BENCH_COUNT: 8 - -jobs: - bench: - name: Default vs SIMD (${{ matrix.runner }}) - # Not every runner pool has AVX-512; several are tried so at least one is - # likely to execute the SIMD backend. - strategy: - fail-fast: false - matrix: - runner: [uci-default, ubuntu-latest] - runs-on: ${{ matrix.runner }} - steps: - # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - fetch-depth: 1 - - - uses: actions/setup-go@v7 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - - - name: Download modules - run: go mod download - - - name: CPU - run: | - lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 - - # Both builds must pass: the default build has no SIMD backend compiled - # in, the experiment build adds it and the differential tests compare - # every available backend against the Blake3 reference. - - name: Test without GOEXPERIMENT=simd - run: go test -count=1 -race ${{ env.LTHASH_PKG }} - - - name: Test with GOEXPERIMENT=simd - env: - GOEXPERIMENT: simd - run: go test -count=1 -race ${{ env.LTHASH_PKG }} - - - name: Benchmark every available backend - env: - GOEXPERIMENT: simd - run: | - go test \ - -run '^$' \ - -bench . \ - -count ${{ env.BENCH_COUNT }} \ - ${{ env.LTHASH_PKG }} | tee bench.txt - - # benchstat groups the samples by the `backend=` sub-benchmark name, so the - # simd column reads as a delta against default. The report is printed to - # the log, added to the job summary and upserted as a PR comment. - - name: Summarise with benchstat - run: | - go install golang.org/x/perf/cmd/benchstat@v0.0.0-20250813145418-2f7363a06fe1 - { - echo '### LtHash default vs SIMD (`${{ matrix.runner }}`)' - echo - echo "CPU: $(lscpu | sed -nE 's/^Model name:\s+//p')" - echo - if grep -q 'backend=simd' bench.txt; then - echo '`HashChunk` is the end-to-end per-block path; `vs base` is simd relative to default.' - else - echo '**This runner CPU lacks AVX-512F + VBMI2, so only the default backend ran.**' - fi - echo - echo '```' - benchstat -col /backend bench.txt - echo '```' - } | tee benchstat.md >> "$GITHUB_STEP_SUMMARY" - if ! grep -q 'backend=simd' bench.txt; then - echo '::warning::SIMD backend unavailable on this runner CPU; only the default backend was benchmarked' - fi - - - name: Post benchstat report on the PR - if: github.event_name == 'pull_request' - continue-on-error: true - uses: actions/github-script@v8 - env: - MARKER: '' - with: - script: | - const fs = require('fs'); - const marker = process.env.MARKER; - const body = `${marker}\n${fs.readFileSync('benchstat.md', 'utf8')}`; - const { owner, repo } = context.repo; - const issue_number = context.issue.number; - const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number, per_page: 100 }); - const existing = comments.find(c => c.body && c.body.startsWith(marker)); - if (existing) { - await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); - } else { - await github.rest.issues.createComment({ owner, repo, issue_number, body }); - } - - - uses: actions/upload-artifact@v4 - with: - name: lthash-bench-${{ matrix.runner }} - path: | - bench.txt - benchstat.md diff --git a/.github/workflows/simd-hash-bench.yml b/.github/workflows/simd-hash-bench.yml new file mode 100644 index 0000000000..46be8a749c --- /dev/null +++ b/.github/workflows/simd-hash-bench.yml @@ -0,0 +1,228 @@ +name: SIMD hash backends +on: + workflow_dispatch: + pull_request: + paths: + - 'sei-db/state_db/sc/flatkv/lthash/**' + - 'sei-tendermint/crypto/merkle/**' + - 'sei-tendermint/crypto/tmhash/**' + - '.github/workflows/simd-hash-bench.yml' + push: + branches: + - main + paths: + - 'sei-db/state_db/sc/flatkv/lthash/**' + - 'sei-tendermint/crypto/merkle/**' + - 'sei-tendermint/crypto/tmhash/**' + +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + +permissions: + contents: read + pull-requests: write + +env: + GO_VERSION: '1.27.1' + LTHASH_PKG: ./sei-db/state_db/sc/flatkv/lthash + BENCH_COUNT: 8 + +jobs: + bench: + name: LtHash default vs SIMD (${{ matrix.runner }}) + # Not every runner pool has AVX-512; several are tried so at least one is + # likely to execute the SIMD backend. + strategy: + fail-fast: false + matrix: + runner: [uci-default, ubuntu-latest] + runs-on: ${{ matrix.runner }} + steps: + # See: https://github.com/actions/checkout/releases/tag/v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 1 + + - uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Download modules + run: go mod download + + - name: CPU + run: | + lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 + + # Both builds must pass: the default build has no SIMD backend compiled + # in, the experiment build adds it and the differential tests compare + # every available backend against the Blake3 reference. + - name: Test without GOEXPERIMENT=simd + run: go test -count=1 -race ${{ env.LTHASH_PKG }} + + - name: Test with GOEXPERIMENT=simd + env: + GOEXPERIMENT: simd + run: go test -count=1 -race ${{ env.LTHASH_PKG }} + + - name: Benchmark every available backend + env: + GOEXPERIMENT: simd + run: | + go test \ + -run '^$' \ + -bench . \ + -count ${{ env.BENCH_COUNT }} \ + ${{ env.LTHASH_PKG }} | tee bench.txt + + # benchstat groups the samples by the `backend=` sub-benchmark name, so the + # simd column reads as a delta against default. The report is printed to + # the log, added to the job summary and upserted as a PR comment. + - name: Summarise with benchstat + run: | + go install golang.org/x/perf/cmd/benchstat@v0.0.0-20250813145418-2f7363a06fe1 + { + echo '### LtHash default vs SIMD (`${{ matrix.runner }}`)' + echo + echo "CPU: $(lscpu | sed -nE 's/^Model name:\s+//p')" + echo + if grep -q 'backend=simd' bench.txt; then + echo '`HashChunk` is the end-to-end per-block path; `vs base` is simd relative to default.' + else + echo '**This runner CPU lacks AVX-512F + VBMI2, so only the default backend ran.**' + fi + echo + echo '```' + benchstat -col /backend bench.txt + echo '```' + } | tee benchstat.md >> "$GITHUB_STEP_SUMMARY" + if ! grep -q 'backend=simd' bench.txt; then + echo '::warning::SIMD backend unavailable on this runner CPU; only the default backend was benchmarked' + fi + + - name: Post benchstat report on the PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v8 + env: + MARKER: '' + with: + script: | + const fs = require('fs'); + const marker = process.env.MARKER; + const body = `${marker}\n${fs.readFileSync('benchstat.md', 'utf8')}`; + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number, per_page: 100 }); + const existing = comments.find(c => c.body && c.body.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } + + - uses: actions/upload-artifact@v4 + with: + name: simd-hash-bench-${{ matrix.runner }} + path: | + bench.txt + benchstat.md + + bench-tmhash: + name: tmhash default vs SIMD (${{ matrix.runner }}) + strategy: + fail-fast: false + matrix: + runner: [uci-default, ubuntu-latest] + runs-on: ${{ matrix.runner }} + defaults: + run: + working-directory: sei-tendermint + steps: + # See: https://github.com/actions/checkout/releases/tag/v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 1 + + - uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Download modules + run: go mod download + + - name: CPU + run: | + lscpu | grep -E 'Model name|^Flags' | sed -E 's/^Flags:\s+/Flags: /' | fold -w 120 + + - name: Test without GOEXPERIMENT=simd + run: go test -count=1 -race ./crypto/tmhash ./crypto/merkle + + - name: Test with GOEXPERIMENT=simd + env: + GOEXPERIMENT: simd + run: go test -count=1 -race ./crypto/tmhash ./crypto/merkle + + # The default column comes from a plain build: with GOEXPERIMENT=simd + # on an AVX-512 CPU the runtime's async preemption restores ZMM + # registers without VZEROUPPER, which slows the legacy-SSE SHA-NI path + # and would skew the baseline. + - name: Benchmark every available backend + run: | + { + go test -run '^$' -bench 'SumBatch|HashFromByteSlices' -count ${{ env.BENCH_COUNT }} ./crypto/tmhash ./crypto/merkle + GOEXPERIMENT=simd go test -run '^$' -bench 'SumBatch|HashFromByteSlices' -count ${{ env.BENCH_COUNT }} ./crypto/tmhash ./crypto/merkle + } | tee bench.txt + + - name: Summarise with benchstat + run: | + go install golang.org/x/perf/cmd/benchstat@v0.0.0-20250813145418-2f7363a06fe1 + { + echo '### tmhash / merkle default vs SIMD (`${{ matrix.runner }}`)' + echo + echo "CPU: $(lscpu | sed -nE 's/^Model name:\s+//p')" + echo + if grep -q 'backend=simd' bench.txt; then + echo '`HashFromByteSlices` is the end-to-end Merkle root; `vs base` is simd relative to default (SHA-NI).' + else + echo '**This runner CPU lacks AVX-512F + VBMI + VBMI2, so only the default backend ran.**' + fi + echo + echo '```' + benchstat -col /backend bench.txt + echo '```' + } | tee benchstat.md >> "$GITHUB_STEP_SUMMARY" + if ! grep -q 'backend=simd' bench.txt; then + echo '::warning::SIMD backend unavailable on this runner CPU; only the default backend was benchmarked' + fi + + - name: Post benchstat report on the PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v8 + env: + MARKER: '' + with: + script: | + const fs = require('fs'); + const marker = process.env.MARKER; + const body = `${marker}\n${fs.readFileSync('sei-tendermint/benchstat.md', 'utf8')}`; + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number, per_page: 100 }); + const existing = comments.find(c => c.body && c.body.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } + + - uses: actions/upload-artifact@v4 + with: + name: tmhash-bench-${{ matrix.runner }} + path: | + sei-tendermint/bench.txt + sei-tendermint/benchstat.md diff --git a/sei-tendermint/crypto/merkle/tree.go b/sei-tendermint/crypto/merkle/tree.go index 0dac5d4757..5b4b75f76a 100644 --- a/sei-tendermint/crypto/merkle/tree.go +++ b/sei-tendermint/crypto/merkle/tree.go @@ -4,14 +4,49 @@ import ( "crypto/sha256" "hash" "math/bits" + + "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/tmhash" ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, -// in the provided order. It follows RFC-6962. +// in the provided order. It follows RFC-6962: a leaf hashes as +// SHA-256(0x00 || item), an inner node as SHA-256(0x01 || left || right), and +// the root is the same 32 bytes whichever tmhash backend is active. func HashFromByteSlices(items [][]byte) []byte { + if lanes := tmhash.BatchLanes(); lanes > 1 && len(items) >= lanes { + return hashFromByteSlicesBatched(items) + } return hashFromByteSlices(sha256.New(), items) } +// hashFromByteSlicesBatched builds the same tree as hashFromByteSlices one +// level at a time, handing every level's independent hashes to +// tmhash.SumBatch in one call. Pairing adjacent nodes left to right and +// carrying an odd trailing node up unchanged yields the RFC-6962 split. +func hashFromByteSlicesBatched(items [][]byte) []byte { + nodes := make([][tmhash.Size]byte, len(items)) + tmhash.SumBatch(leafPrefix, items, nodes) + pairs := make([]byte, len(items)/2*2*tmhash.Size) + msgs := make([][]byte, len(items)/2) + for len(nodes) > 1 { + np := len(nodes) / 2 + for i := range np { + pair := pairs[i*2*tmhash.Size : (i+1)*2*tmhash.Size] + copy(pair, nodes[2*i][:]) + copy(pair[tmhash.Size:], nodes[2*i+1][:]) + msgs[i] = pair + } + carry := len(nodes) % 2 + if carry == 1 { + nodes[np] = nodes[len(nodes)-1] + } + tmhash.SumBatch(innerPrefix, msgs[:np], nodes[:np]) + nodes = nodes[:np+carry] + } + root := nodes[0] + return root[:] +} + func hashFromByteSlices(sha hash.Hash, items [][]byte) []byte { switch len(items) { case 0: diff --git a/sei-tendermint/crypto/merkle/tree_test.go b/sei-tendermint/crypto/merkle/tree_test.go index d558cc8c9b..7280f99bc5 100644 --- a/sei-tendermint/crypto/merkle/tree_test.go +++ b/sei-tendermint/crypto/merkle/tree_test.go @@ -2,6 +2,7 @@ package merkle import ( "bytes" + "crypto/sha256" "encoding/binary" "encoding/hex" "io" @@ -156,6 +157,64 @@ func BenchmarkHashAlternatives(b *testing.B) { }) } +// TestHashFromByteSlicesBatched checks the level-batched tree against the +// recursive reference around every lane-count boundary. +func TestHashFromByteSlicesBatched(t *testing.T) { + sha := sha256.New() + for _, size := range []int{0, 2, 32, 128} { + for total := 1; total <= 130; total++ { + items := make([][]byte, total) + for i := range items { + items[i] = tmrand.Bytes(size) + } + require.Equal(t, hashFromByteSlices(sha, items), hashFromByteSlicesBatched(items), "size=%d total=%d", size, total) + } + } +} + +// BenchmarkHashFromByteSlices measures the whole tree for tx-hash sized and +// tx sized leaves. The sub-benchmark is named after the active tmhash +// backend so runs under different SEI_TMHASH_BACKEND values can be compared +// with benchstat. +func BenchmarkHashFromByteSlices(b *testing.B) { + for _, tc := range []struct { + name string + total int + size int + }{ + {"leaves=1024/leaf=32", 1024, 32}, + {"leaves=1024/leaf=512", 1024, 512}, + {"leaves=100/leaf=32", 100, 32}, + } { + items := make([][]byte, tc.total) + for i := range items { + items[i] = tmrand.Bytes(tc.size) + } + b.Run(tc.name+"/backend="+tmhash.ActiveBackend(), func(b *testing.B) { + b.SetBytes(int64(tc.total * tc.size)) + for b.Loop() { + _ = HashFromByteSlices(items) + } + }) + } +} + +// BenchmarkHashFromByteSlicesBatched forces the level-batched tree so that, +// pinned to the default backend, it isolates the restructuring from the SIMD +// kernel. +func BenchmarkHashFromByteSlicesBatched(b *testing.B) { + items := make([][]byte, 1024) + for i := range items { + items[i] = tmrand.Bytes(32) + } + b.Run("leaves=1024/leaf=32/backend="+tmhash.ActiveBackend(), func(b *testing.B) { + b.SetBytes(int64(len(items) * 32)) + for b.Loop() { + _ = hashFromByteSlicesBatched(items) + } + }) +} + func Test_getSplitPoint(t *testing.T) { tests := []struct { length int64 diff --git a/sei-tendermint/crypto/tmhash/backend.go b/sei-tendermint/crypto/tmhash/backend.go new file mode 100644 index 0000000000..339c6823f7 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend.go @@ -0,0 +1,60 @@ +package tmhash + +import "os" + +// BackendEnv is the environment variable that pins the batch hashing backend +// by name. An unknown or empty value leaves the selection automatic. +const BackendEnv = "SEI_TMHASH_BACKEND" + +// backend is one implementation of batched SHA-256. Every backend produces +// bit-identical digests; they differ only in how fast they get there. +type backend struct { + name string + // lanes is the number of equal-length messages the backend hashes at + // once; 1 means one message at a time. + lanes int + // sumBatch writes SHA-256(prefix || msgs[i]) to out[i] for every i. + sumBatch func(prefix []byte, msgs [][]byte, out [][Size]byte) +} + +var active = selectBackend(os.Getenv(BackendEnv)) + +// ActiveBackend returns the name of the batch hashing backend in use. +func ActiveBackend() string { + return active.name +} + +// BatchLanes returns how many messages the active backend hashes in parallel. +// Callers batching work should hand over multiples of this many messages. +func BatchLanes() int { + return active.lanes +} + +// SumBatch writes SHA-256(prefix || msgs[i]) to out[i] for every i, as the +// 32-byte big-endian digest crypto/sha256 would produce for the same bytes, +// regardless of the active backend. out must be at least as long as msgs. +func SumBatch(prefix []byte, msgs [][]byte, out [][Size]byte) { + active.sumBatch(prefix, msgs, out) +} + +// availableBackends returns every backend this binary can run on this CPU, +// keyed by name. +func availableBackends() map[string]backend { + m := map[string]backend{defaultBackend.name: defaultBackend} + if b, ok := simdBackend(); ok { + m[b.name] = b + } + return m +} + +// selectBackend picks the backend named by pin, or the fastest available one +// when pin is empty or unknown. +func selectBackend(pin string) backend { + if b, ok := availableBackends()[pin]; ok { + return b + } + if b, ok := simdBackend(); ok { + return b + } + return defaultBackend +} diff --git a/sei-tendermint/crypto/tmhash/backend_default.go b/sei-tendermint/crypto/tmhash/backend_default.go new file mode 100644 index 0000000000..8211c23edd --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_default.go @@ -0,0 +1,39 @@ +package tmhash + +import ( + "crypto/sha256" + "hash" +) + +// defaultBackend hashes one message at a time with crypto/sha256, which uses +// the SHA-NI single-lane instructions where the CPU has them. +var defaultBackend = backend{ + name: "default", + lanes: 1, + sumBatch: sumBatchScalar, +} + +func sumBatchScalar(prefix []byte, msgs [][]byte, out [][Size]byte) { + h := sha256.New() + for i, msg := range msgs { + sumOne(h, prefix, msg, &out[i]) + } +} + +// sumScalarAt hashes msgs[i] into out[i] for every i in idx. +func sumScalarAt(prefix []byte, msgs [][]byte, out [][Size]byte, idx []int) { + if len(idx) == 0 { + return + } + h := sha256.New() + for _, i := range idx { + sumOne(h, prefix, msgs[i], &out[i]) + } +} + +func sumOne(h hash.Hash, prefix, msg []byte, out *[Size]byte) { + h.Reset() + h.Write(prefix) + h.Write(msg) + h.Sum(out[:0]) +} diff --git a/sei-tendermint/crypto/tmhash/backend_nosimd.go b/sei-tendermint/crypto/tmhash/backend_nosimd.go new file mode 100644 index 0000000000..aa4ceec377 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_nosimd.go @@ -0,0 +1,7 @@ +//go:build !(goexperiment.simd && amd64) + +package tmhash + +func simdBackend() (backend, bool) { + return backend{}, false +} diff --git a/sei-tendermint/crypto/tmhash/backend_simd_amd64.go b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go new file mode 100644 index 0000000000..88db55ff18 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_simd_amd64.go @@ -0,0 +1,263 @@ +//go:build goexperiment.simd && amd64 + +package tmhash + +import ( + "encoding/binary" + "sync" + + "simd/archsimd" +) + +//go:generate go run gen_sha256x16.go + +// vzeroupper clears the upper halves of the vector registers. +// +//go:noescape +func vzeroupper() + +const ( + simdBackendName = "simd" + // simdLanes is the number of 32-bit SHA-256 words in one 512-bit ZMM + // register, i.e. how many independent messages one kernel pass hashes. + simdLanes = 16 + blockSize = 64 + // simdMaxBlocks bounds the per-lane message length hashed by the SIMD + // kernel; longer messages go through the scalar path so the block + // scratch stays small. + simdMaxBlocks = 64 +) + +// simdBackend returns the AVX-512 backend when the CPU can run it. AVX512 +// covers F/BW/DQ/VL; VBMI provides VPERMB for the prefix shift and VBMI2 +// VPSHRDD for the rotates. +func simdBackend() (backend, bool) { + if !archsimd.X86.AVX512() || !archsimd.X86.AVX512VBMI() || !archsimd.X86.AVX512VBMI2() { + return backend{}, false + } + return backend{ + name: simdBackendName, + lanes: simdLanes, + sumBatch: sumBatchSIMD, + }, true +} + +// laneScratch is the per-call working set: the transposed message blocks fed +// to the kernel and, per lane, the final padding block's big-endian bit length +// in bytes 56-63 (the other bytes stay zero). +type laneScratch struct { + blocks []sha256Block16 + lens [simdLanes][blockSize]byte + pfx prefixShift +} + +func (sp *laneScratch) setPrefix(prefix []byte) { + if len(prefix) > 0 { + sp.pfx.init(prefix) + } +} + +var laneScratchPool = sync.Pool{ + New: func() any { return &laneScratch{blocks: make([]sha256Block16, 4)} }, +} + +// paddedBlocks returns how many 64-byte blocks SHA-256 processes for a message +// of n bytes: the message, a 0x80 byte and a 64-bit length. +func paddedBlocks(n int) int { + return (n + 1 + 8 + blockSize - 1) / blockSize +} + +// sumBatchSIMD hashes sixteen messages of equal padded block count at a time +// and hands whatever does not fill a group of sixteen to the scalar backend. +func sumBatchSIMD(prefix []byte, msgs [][]byte, out [][Size]byte) { + if len(msgs) < simdLanes || len(prefix) >= blockSize { + sumBatchScalar(prefix, msgs, out) + return + } + sp := laneScratchPool.Get().(*laneScratch) + sp.setPrefix(prefix) + // Bucket message indices by block count. Merkle levels are either all + // inner nodes or leaves of similar size, so most calls stay on the + // single-bucket path. + var buckets map[int][]int + var rest []int + var lanes [simdLanes]int + first := paddedBlocks(len(prefix) + len(msgs[0])) + next := 0 + for i, msg := range msgs { + nb := paddedBlocks(len(prefix) + len(msg)) + if nb == first && nb <= simdMaxBlocks { + lanes[next] = i + if next++; next == simdLanes { + sha256Lanes(sp, prefix, msgs, out, &lanes, nb) + next = 0 + } + continue + } + if nb > simdMaxBlocks { + rest = append(rest, i) + continue + } + if buckets == nil { + buckets = map[int][]int{} + } + buckets[nb] = append(buckets[nb], i) + } + rest = append(rest, lanes[:next]...) + for nb, idx := range buckets { + for len(idx) >= simdLanes { + copy(lanes[:], idx[:simdLanes]) + idx = idx[simdLanes:] + sha256Lanes(sp, prefix, msgs, out, &lanes, nb) + } + rest = append(rest, idx...) + } + laneScratchPool.Put(sp) + // The compiler emits no VZEROUPPER after AVX-512 code, and the legacy-SSE + // SHA-NI path below runs several times slower while the upper halves are + // dirty. + vzeroupper() + sumScalarAt(prefix, msgs, out, rest) +} + +// sha256Lanes hashes the sixteen messages selected by lanes, each of nb +// padded blocks, with one kernel call. Each lane's blocks are the standard +// SHA-256 message layout, prefix || msg || 0x80 || zeros || 64-bit big-endian +// bit length, transposed into sha256Block16 word-major form; the state words +// are read back per lane and stored big-endian as the digest. +func sha256Lanes(sp *laneScratch, prefix []byte, msgs [][]byte, out [][Size]byte, lanes *[simdLanes]int, nb int) { + if cap(sp.blocks) < nb { + sp.blocks = make([]sha256Block16, nb) + } + blocks := sp.blocks[:nb] + for lane, i := range lanes { + n := len(prefix) + len(msgs[i]) + binary.BigEndian.PutUint64(sp.lens[lane][blockSize-8:], uint64(n)*8) //nolint:gosec // G115 n is a slice length + } + pfx := &sp.pfx + for b := range blocks { + start := b * blockSize + var rows [simdLanes]archsimd.Uint8x64 + for lane, i := range lanes { + msg := msgs[i] + var row archsimd.Uint8x64 + if start >= len(prefix) { + row, _ = archsimd.LoadUint8x64Part(msg[min(start-len(prefix), len(msg)):]) + } else { + m, _ := archsimd.LoadUint8x64Part(msg) + row = m.Permute(pfx.shift).And(pfx.keep).Or(pfx.bytes) + } + if p := len(prefix) + len(msg) - start; p >= 0 && p < blockSize { + row = row.Or(archsimd.LoadUint8x64Array(&pad80[p])) + } + if b == nb-1 { + row = row.Or(archsimd.LoadUint8x64Array(&sp.lens[lane])) + } + rows[lane] = row + } + transposeBlock(&rows, &blocks[b]) + } + var st [8][16]uint32 + sha256x16(blocks, &st) + for lane, i := range lanes { + d := &out[i] + for w := range 8 { + binary.BigEndian.PutUint32(d[4*w:], st[w][lane]) + } + } +} + +// prefixShift assembles the first block of prefix || msg from a raw load of +// msg: shift moves msg byte i to position i+len(prefix), keep zeroes the +// prefix positions and bytes holds the prefix itself. The prefix must be +// shorter than a block. +type prefixShift struct { + shift, keep, bytes archsimd.Uint8x64 +} + +func (p *prefixShift) init(prefix []byte) { + var shift, keep, bytes [blockSize]byte + for i := range blockSize { + if i < len(prefix) { + bytes[i] = prefix[i] + continue + } + shift[i] = byte(i - len(prefix)) //nolint:gosec // G115 0 <= i-len(prefix) < 64 + keep[i] = 0xff + } + p.shift = archsimd.LoadUint8x64Array(&shift) + p.keep = archsimd.LoadUint8x64Array(&keep) + p.bytes = archsimd.LoadUint8x64Array(&bytes) +} + +// pad80[p] is a block with the SHA-256 terminator byte at offset p. +var pad80 = func() [blockSize][blockSize]byte { + var t [blockSize][blockSize]byte + for p := range t { + t[p][p] = 0x80 + } + return t +}() + +// bswap32 reverses the bytes of every 32-bit word (VPSHUFB within 128-bit +// groups); SHA-256 words are big-endian. +var bswap32 = func() [64]int8 { + var idx [64]int8 + for i := range idx { + idx[i] = int8((i &^ 3) + (3 - i&3)) //nolint:gosec // G115 0 <= i < 64 + } + return idx +}() + +// transposeIdx holds, per stage k, the ConcatPermute index vectors that swap +// bit k of the row index with bit k of the element index: [k][0] produces the +// row with bit k clear, [k][1] the row with bit k set. Indices 16-31 select +// from the second source. +var transposeIdx = func() [4][2][16]uint32 { + var idx [4][2][16]uint32 + for k := range 4 { + bk := uint32(1) << k + for e := range uint32(16) { + if e&bk == 0 { + idx[k][0][e] = e + idx[k][1][e] = e | bk + } else { + idx[k][0][e] = 16 + (e ^ bk) + idx[k][1][e] = 16 + e + } + } + } + return idx +}() + +// bswapRows reinterprets each 64-byte row as sixteen big-endian words. +func bswapRows(rows *[simdLanes]archsimd.Uint8x64, v *[16]archsimd.Uint32x16) { + sw := archsimd.LoadInt8x64Array(&bswap32) + for i := range v { + v[i] = rows[i].PermuteOrZeroGrouped(sw).ReshapeToUint32s() + } +} + +// transposeBlock converts rows[lane] (64 message bytes of one lane) into +// blk[word][lane] big-endian words with a four-stage in-register transpose. +func transposeBlock(rows *[simdLanes]archsimd.Uint8x64, blk *sha256Block16) { + var v [16]archsimd.Uint32x16 + bswapRows(rows, &v) + for k := range 4 { + lo := archsimd.LoadUint32x16Array(&transposeIdx[k][0]) + hi := archsimd.LoadUint32x16Array(&transposeIdx[k][1]) + bk := 1 << k + for i := range 16 { + if i&bk != 0 { + continue + } + j := i | bk + x, y := v[i], v[j] + v[i] = x.ConcatPermute(y, lo) + v[j] = x.ConcatPermute(y, hi) + } + } + for w := range v { + v[w].StoreArray(&blk[w]) + } +} diff --git a/sei-tendermint/crypto/tmhash/backend_test.go b/sei-tendermint/crypto/tmhash/backend_test.go new file mode 100644 index 0000000000..c75112e67d --- /dev/null +++ b/sei-tendermint/crypto/tmhash/backend_test.go @@ -0,0 +1,141 @@ +package tmhash + +import ( + "crypto/sha256" + "fmt" + "maps" + "slices" + "testing" + + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" +) + +// batchSizes covers SHA-256 block boundaries with and without the one-byte +// prefix, the 65-byte Merkle inner node and a few multi-block messages. +var batchSizes = []int{0, 1, 31, 32, 54, 55, 56, 63, 64, 65, 100, 118, 119, 120, 127, 128, 129, 200, 1000, 4096, 5000} + +func availableBackendNames() []string { + return slices.Sorted(maps.Keys(availableBackends())) +} + +func referenceSum(prefix, msg []byte) [Size]byte { + return sha256.Sum256(slices.Concat(prefix, msg)) +} + +func randomMsgs(rng utils.Rng, n, size int) [][]byte { + msgs := make([][]byte, n) + for i := range msgs { + msgs[i] = utils.GenBytes(rng, size) + } + return msgs +} + +func TestBackendsAgreeWithReference(t *testing.T) { + for _, name := range availableBackendNames() { + b := availableBackends()[name] + for _, prefix := range [][]byte{nil, {0}, {1}} { + for _, size := range batchSizes { + // One partial batch, one exact multiple, one with a remainder. + for _, n := range []int{1, 15, 16, 32, 37} { + t.Run(fmt.Sprintf("%s/prefix=%d/size=%d/n=%d", name, len(prefix), size, n), func(t *testing.T) { + rng := utils.TestRng() + msgs := randomMsgs(rng, n, size) + out := make([][Size]byte, n) + b.sumBatch(prefix, msgs, out) + for i, msg := range msgs { + require.Equal(t, referenceSum(prefix, msg), out[i]) + } + }) + } + } + } + } +} + +// checkBatch hashes msgs with every available backend and compares each +// digest with crypto/sha256. +func checkBatch(t testing.TB, prefix []byte, msgs [][]byte) { + for _, name := range availableBackendNames() { + out := make([][Size]byte, len(msgs)) + availableBackends()[name].sumBatch(prefix, msgs, out) + for i, msg := range msgs { + require.Equal(t, referenceSum(prefix, msg), out[i], "%s msg %d len %d", name, i, len(msg)) + } + } +} + +// TestBackendsAgreeOnMixedSizes mixes, in one call, sizes that fill SIMD +// lanes, sizes that land in the smaller buckets and a few messages beyond +// the SIMD kernel's block limit. +func TestBackendsAgreeOnMixedSizes(t *testing.T) { + rng := utils.TestRng() + msgs := make([][]byte, 200) + for i := range msgs { + size := rng.Intn(600) + if rng.Intn(20) == 0 { + size = 4096 + rng.Intn(2000) + } + msgs[i] = utils.GenBytes(rng, size) + } + checkBatch(t, []byte{0}, msgs) +} + +// FuzzSumBatch drives every backend with an arbitrary prefix and message +// length list against crypto/sha256. Each byte of lens is one message whose +// length is the byte value scaled by 24, so that lengths span from 0 to +// beyond the SIMD kernel's block limit. +func FuzzSumBatch(f *testing.F) { + f.Add([]byte{0}, []byte{2, 3, 200, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}) + f.Add([]byte{}, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + f.Fuzz(func(t *testing.T, prefix, lens []byte) { + if len(prefix) > 63 { + prefix = prefix[:63] + } + rng := utils.TestRng() + msgs := make([][]byte, len(lens)) + for i, l := range lens { + msgs[i] = utils.GenBytes(rng, int(l)*24) + } + checkBatch(t, prefix, msgs) + }) +} + +func TestSelectBackend(t *testing.T) { + require.Equal(t, "default", selectBackend("default").name) + require.Equal(t, 1, selectBackend("default").lanes) + auto := selectBackend("") + require.Equal(t, auto.name, selectBackend("unknown").name) + if simd, ok := simdBackend(); ok { + require.Equal(t, simd.name, auto.name) + require.Equal(t, simd.name, selectBackend(simd.name).name) + } else { + require.Equal(t, "default", auto.name) + } + require.True(t, slices.Contains(availableBackendNames(), ActiveBackend())) +} + +// benchmarkSumBatch measures the active backend only, named after it so runs +// under different SEI_TMHASH_BACKEND values or builds compare with benchstat +// without one build's scalar samples polluting the other's column. +func benchmarkSumBatch(b *testing.B, size, n int) { + msgs := randomMsgs(utils.TestRng(), n, size) + out := make([][Size]byte, n) + prefix := []byte{0} + b.Run("backend="+ActiveBackend(), func(b *testing.B) { + b.SetBytes(int64(n * (size + 1))) + for b.Loop() { + SumBatch(prefix, msgs, out) + } + }) +} + +// BenchmarkSumBatchInner is a Merkle inner-node level: 1024 x 64-byte +// messages behind a one-byte prefix. +func BenchmarkSumBatchInner(b *testing.B) { benchmarkSumBatch(b, 64, 1024) } + +// BenchmarkSumBatchLeaf256 is a leaf level of 1024 x 256-byte items. +func BenchmarkSumBatchLeaf256(b *testing.B) { benchmarkSumBatch(b, 256, 1024) } + +// BenchmarkSumBatchLeaf1K is a leaf level of 1024 x 1 KiB items. +func BenchmarkSumBatchLeaf1K(b *testing.B) { benchmarkSumBatch(b, 1024, 1024) } diff --git a/sei-tendermint/crypto/tmhash/gen_sha256x16.go b/sei-tendermint/crypto/tmhash/gen_sha256x16.go new file mode 100644 index 0000000000..308205c3c6 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/gen_sha256x16.go @@ -0,0 +1,138 @@ +//go:build ignore + +// gen_sha256x16 writes sha256x16_amd64.go: a fully unrolled 16-lane SHA-256 +// compression over simd/archsimd Uint32x16 vectors. Each lane is one message; +// all lanes must have the same number of padded blocks. +// +// Usage: go run gen_sha256x16.go +package main + +import ( + "bytes" + "fmt" + "go/format" + "os" +) + +var k = [64]uint32{ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +} + +var h0 = [8]uint32{ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +} + +func main() { + var b bytes.Buffer + p := func(format string, args ...any) { fmt.Fprintf(&b, format+"\n", args...) } + + p("// Code generated by gen_sha256x16.go; DO NOT EDIT.") + p("") + p("//go:build goexperiment.simd && amd64") + p("") + p("package tmhash") + p("") + p(`import "simd/archsimd"`) + p("") + p("// sha256Block16 is one 64-byte block of each of 16 messages, laid out") + p("// block16[word][lane] so that a word of every lane loads as one vector.") + p("type sha256Block16 [16][16]uint32") + p("") + p("// Constants are kept pre-broadcast and loaded with LoadUint32x16Array;") + p("// archsimd.BroadcastUint32x16 compiles to a legacy-SSE sequence that costs") + p("// an SSE/AVX transition on every call.") + p("var (") + p("\tsha256K16 [64][16]uint32") + p("\tsha256H16 [8][16]uint32") + p(")") + p("") + p("func init() {") + p("\tfor t, v := range [64]uint32{") + for t := 0; t < 64; t += 8 { + p("\t\t0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x,", + k[t], k[t+1], k[t+2], k[t+3], k[t+4], k[t+5], k[t+6], k[t+7]) + } + p("\t} {") + p("\t\tfor lane := range sha256K16[t] {") + p("\t\t\tsha256K16[t][lane] = v") + p("\t\t}") + p("\t}") + p("\tfor i, v := range [8]uint32{") + p("\t\t0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x,", + h0[0], h0[1], h0[2], h0[3], h0[4], h0[5], h0[6], h0[7]) + p("\t} {") + p("\t\tfor lane := range sha256H16[i] {") + p("\t\t\tsha256H16[i][lane] = v") + p("\t\t}") + p("\t}") + p("}") + p("") + p("// rotr32 rotates every lane right by n using VPSHRDD; RotateAllRight is") + p("// emulated with three instructions.") + p("func rotr32(x archsimd.Uint32x16, n uint64) archsimd.Uint32x16 {") + p("\treturn x.ShiftAllRightConcatMod32(x, n)") + p("}") + p("") + p("// sha256x16 compresses blocks, one sha256Block16 per message block, and") + p("// writes the final state as out[word][lane] (big-endian digest words).") + p("func sha256x16(blocks []sha256Block16, out *[8][16]uint32) {") + for i := range 8 { + p("\th%d := archsimd.LoadUint32x16Array(&sha256H16[%d])", i, i) + } + p("\tfor bi := range blocks {") + p("\t\tblk := &blocks[bi]") + for t := range 16 { + p("\t\tw%d := archsimd.LoadUint32x16Array(&blk[%d])", t, t) + } + names := [8]string{"h0", "h1", "h2", "h3", "h4", "h5", "h6", "h7"} + // Working variables a..h are renamed each round instead of shuffled. + v := [8]string{"a", "b", "c", "d", "e", "f", "g", "h"} + for i := range v { + p("\t\t%s := %s", v[i], names[i]) + } + p("\t\tvar t1, t2 archsimd.Uint32x16") + for t := range 64 { + if t >= 16 { + w16, w15, w7, w2 := t%16, (t-15)%16, (t-7)%16, (t-2)%16 + p("\t\tw%d = w%d.Add(rotr32(w%d, 7).Xor(rotr32(w%d, 18)).Xor(w%d.ShiftAllRight(3))).Add(w%d).Add(rotr32(w%d, 17).Xor(rotr32(w%d, 19)).Xor(w%d.ShiftAllRight(10)))", + w16, w16, w15, w15, w15, w7, w2, w2, w2) + } + a, bb, c, d, e, f, g, hh := v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7] + // T1 = h + Σ1(e) + Ch(e,f,g) + K[t] + W[t] + p("\t\tt1 = %s.Add(rotr32(%s, 6).Xor(rotr32(%s, 11)).Xor(rotr32(%s, 25))).Add(%s.And(%s).Xor(%s.AndNot(%s))).Add(archsimd.LoadUint32x16Array(&sha256K16[%d])).Add(w%d)", + hh, e, e, e, e, f, g, e, t, t%16) + // T2 = Σ0(a) + Maj(a,b,c), Maj = (a&b) ^ ((a^b)&c) + p("\t\tt2 = rotr32(%s, 2).Xor(rotr32(%s, 13)).Xor(rotr32(%s, 22)).Add(%s.And(%s).Xor(%s.Xor(%s).And(%s)))", + a, a, a, a, bb, a, bb, c) + // h <- g, g <- f, f <- e, e <- d+T1, d <- c, c <- b, b <- a, a <- T1+T2. + // Rotating the names makes old h the new a and old d the new e. + p("\t\t%s = %s.Add(t1)", d, d) + p("\t\t%s = t1.Add(t2)", hh) + v = [8]string{hh, a, bb, c, d, e, f, g} + } + for i := range v { + p("\t\t%s = %s.Add(%s)", names[i], names[i], v[i]) + } + p("\t}") + for i := range 8 { + p("\th%d.StoreArray(&out[%d])", i, i) + } + p("}") + + src, err := format.Source(b.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := os.WriteFile("sha256x16_amd64.go", src, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/sei-tendermint/crypto/tmhash/sha256x16_amd64.go b/sei-tendermint/crypto/tmhash/sha256x16_amd64.go new file mode 100644 index 0000000000..3f8d1229c7 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/sha256x16_amd64.go @@ -0,0 +1,410 @@ +// Code generated by gen_sha256x16.go; DO NOT EDIT. + +//go:build goexperiment.simd && amd64 + +package tmhash + +import "simd/archsimd" + +// sha256Block16 is one 64-byte block of each of 16 messages, laid out +// block16[word][lane] so that a word of every lane loads as one vector. +type sha256Block16 [16][16]uint32 + +// Constants are kept pre-broadcast and loaded with LoadUint32x16Array; +// archsimd.BroadcastUint32x16 compiles to a legacy-SSE sequence that costs +// an SSE/AVX transition on every call. +var ( + sha256K16 [64][16]uint32 + sha256H16 [8][16]uint32 +) + +func init() { + for t, v := range [64]uint32{ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + } { + for lane := range sha256K16[t] { + sha256K16[t][lane] = v + } + } + for i, v := range [8]uint32{ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + } { + for lane := range sha256H16[i] { + sha256H16[i][lane] = v + } + } +} + +// rotr32 rotates every lane right by n using VPSHRDD; RotateAllRight is +// emulated with three instructions. +func rotr32(x archsimd.Uint32x16, n uint64) archsimd.Uint32x16 { + return x.ShiftAllRightConcatMod32(x, n) +} + +// sha256x16 compresses blocks, one sha256Block16 per message block, and +// writes the final state as out[word][lane] (big-endian digest words). +func sha256x16(blocks []sha256Block16, out *[8][16]uint32) { + h0 := archsimd.LoadUint32x16Array(&sha256H16[0]) + h1 := archsimd.LoadUint32x16Array(&sha256H16[1]) + h2 := archsimd.LoadUint32x16Array(&sha256H16[2]) + h3 := archsimd.LoadUint32x16Array(&sha256H16[3]) + h4 := archsimd.LoadUint32x16Array(&sha256H16[4]) + h5 := archsimd.LoadUint32x16Array(&sha256H16[5]) + h6 := archsimd.LoadUint32x16Array(&sha256H16[6]) + h7 := archsimd.LoadUint32x16Array(&sha256H16[7]) + for bi := range blocks { + blk := &blocks[bi] + w0 := archsimd.LoadUint32x16Array(&blk[0]) + w1 := archsimd.LoadUint32x16Array(&blk[1]) + w2 := archsimd.LoadUint32x16Array(&blk[2]) + w3 := archsimd.LoadUint32x16Array(&blk[3]) + w4 := archsimd.LoadUint32x16Array(&blk[4]) + w5 := archsimd.LoadUint32x16Array(&blk[5]) + w6 := archsimd.LoadUint32x16Array(&blk[6]) + w7 := archsimd.LoadUint32x16Array(&blk[7]) + w8 := archsimd.LoadUint32x16Array(&blk[8]) + w9 := archsimd.LoadUint32x16Array(&blk[9]) + w10 := archsimd.LoadUint32x16Array(&blk[10]) + w11 := archsimd.LoadUint32x16Array(&blk[11]) + w12 := archsimd.LoadUint32x16Array(&blk[12]) + w13 := archsimd.LoadUint32x16Array(&blk[13]) + w14 := archsimd.LoadUint32x16Array(&blk[14]) + w15 := archsimd.LoadUint32x16Array(&blk[15]) + a := h0 + b := h1 + c := h2 + d := h3 + e := h4 + f := h5 + g := h6 + h := h7 + var t1, t2 archsimd.Uint32x16 + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[0])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[1])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[2])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[3])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[4])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[5])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[6])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[7])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[8])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[9])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[10])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[11])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[12])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[13])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[14])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[15])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w0 = w0.Add(rotr32(w1, 7).Xor(rotr32(w1, 18)).Xor(w1.ShiftAllRight(3))).Add(w9).Add(rotr32(w14, 17).Xor(rotr32(w14, 19)).Xor(w14.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[16])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w1 = w1.Add(rotr32(w2, 7).Xor(rotr32(w2, 18)).Xor(w2.ShiftAllRight(3))).Add(w10).Add(rotr32(w15, 17).Xor(rotr32(w15, 19)).Xor(w15.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[17])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w2 = w2.Add(rotr32(w3, 7).Xor(rotr32(w3, 18)).Xor(w3.ShiftAllRight(3))).Add(w11).Add(rotr32(w0, 17).Xor(rotr32(w0, 19)).Xor(w0.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[18])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w3 = w3.Add(rotr32(w4, 7).Xor(rotr32(w4, 18)).Xor(w4.ShiftAllRight(3))).Add(w12).Add(rotr32(w1, 17).Xor(rotr32(w1, 19)).Xor(w1.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[19])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w4 = w4.Add(rotr32(w5, 7).Xor(rotr32(w5, 18)).Xor(w5.ShiftAllRight(3))).Add(w13).Add(rotr32(w2, 17).Xor(rotr32(w2, 19)).Xor(w2.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[20])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w5 = w5.Add(rotr32(w6, 7).Xor(rotr32(w6, 18)).Xor(w6.ShiftAllRight(3))).Add(w14).Add(rotr32(w3, 17).Xor(rotr32(w3, 19)).Xor(w3.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[21])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w6 = w6.Add(rotr32(w7, 7).Xor(rotr32(w7, 18)).Xor(w7.ShiftAllRight(3))).Add(w15).Add(rotr32(w4, 17).Xor(rotr32(w4, 19)).Xor(w4.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[22])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w7 = w7.Add(rotr32(w8, 7).Xor(rotr32(w8, 18)).Xor(w8.ShiftAllRight(3))).Add(w0).Add(rotr32(w5, 17).Xor(rotr32(w5, 19)).Xor(w5.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[23])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w8 = w8.Add(rotr32(w9, 7).Xor(rotr32(w9, 18)).Xor(w9.ShiftAllRight(3))).Add(w1).Add(rotr32(w6, 17).Xor(rotr32(w6, 19)).Xor(w6.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[24])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w9 = w9.Add(rotr32(w10, 7).Xor(rotr32(w10, 18)).Xor(w10.ShiftAllRight(3))).Add(w2).Add(rotr32(w7, 17).Xor(rotr32(w7, 19)).Xor(w7.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[25])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w10 = w10.Add(rotr32(w11, 7).Xor(rotr32(w11, 18)).Xor(w11.ShiftAllRight(3))).Add(w3).Add(rotr32(w8, 17).Xor(rotr32(w8, 19)).Xor(w8.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[26])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w11 = w11.Add(rotr32(w12, 7).Xor(rotr32(w12, 18)).Xor(w12.ShiftAllRight(3))).Add(w4).Add(rotr32(w9, 17).Xor(rotr32(w9, 19)).Xor(w9.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[27])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w12 = w12.Add(rotr32(w13, 7).Xor(rotr32(w13, 18)).Xor(w13.ShiftAllRight(3))).Add(w5).Add(rotr32(w10, 17).Xor(rotr32(w10, 19)).Xor(w10.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[28])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w13 = w13.Add(rotr32(w14, 7).Xor(rotr32(w14, 18)).Xor(w14.ShiftAllRight(3))).Add(w6).Add(rotr32(w11, 17).Xor(rotr32(w11, 19)).Xor(w11.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[29])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w14 = w14.Add(rotr32(w15, 7).Xor(rotr32(w15, 18)).Xor(w15.ShiftAllRight(3))).Add(w7).Add(rotr32(w12, 17).Xor(rotr32(w12, 19)).Xor(w12.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[30])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w15 = w15.Add(rotr32(w0, 7).Xor(rotr32(w0, 18)).Xor(w0.ShiftAllRight(3))).Add(w8).Add(rotr32(w13, 17).Xor(rotr32(w13, 19)).Xor(w13.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[31])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w0 = w0.Add(rotr32(w1, 7).Xor(rotr32(w1, 18)).Xor(w1.ShiftAllRight(3))).Add(w9).Add(rotr32(w14, 17).Xor(rotr32(w14, 19)).Xor(w14.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[32])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w1 = w1.Add(rotr32(w2, 7).Xor(rotr32(w2, 18)).Xor(w2.ShiftAllRight(3))).Add(w10).Add(rotr32(w15, 17).Xor(rotr32(w15, 19)).Xor(w15.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[33])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w2 = w2.Add(rotr32(w3, 7).Xor(rotr32(w3, 18)).Xor(w3.ShiftAllRight(3))).Add(w11).Add(rotr32(w0, 17).Xor(rotr32(w0, 19)).Xor(w0.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[34])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w3 = w3.Add(rotr32(w4, 7).Xor(rotr32(w4, 18)).Xor(w4.ShiftAllRight(3))).Add(w12).Add(rotr32(w1, 17).Xor(rotr32(w1, 19)).Xor(w1.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[35])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w4 = w4.Add(rotr32(w5, 7).Xor(rotr32(w5, 18)).Xor(w5.ShiftAllRight(3))).Add(w13).Add(rotr32(w2, 17).Xor(rotr32(w2, 19)).Xor(w2.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[36])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w5 = w5.Add(rotr32(w6, 7).Xor(rotr32(w6, 18)).Xor(w6.ShiftAllRight(3))).Add(w14).Add(rotr32(w3, 17).Xor(rotr32(w3, 19)).Xor(w3.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[37])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w6 = w6.Add(rotr32(w7, 7).Xor(rotr32(w7, 18)).Xor(w7.ShiftAllRight(3))).Add(w15).Add(rotr32(w4, 17).Xor(rotr32(w4, 19)).Xor(w4.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[38])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w7 = w7.Add(rotr32(w8, 7).Xor(rotr32(w8, 18)).Xor(w8.ShiftAllRight(3))).Add(w0).Add(rotr32(w5, 17).Xor(rotr32(w5, 19)).Xor(w5.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[39])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w8 = w8.Add(rotr32(w9, 7).Xor(rotr32(w9, 18)).Xor(w9.ShiftAllRight(3))).Add(w1).Add(rotr32(w6, 17).Xor(rotr32(w6, 19)).Xor(w6.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[40])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w9 = w9.Add(rotr32(w10, 7).Xor(rotr32(w10, 18)).Xor(w10.ShiftAllRight(3))).Add(w2).Add(rotr32(w7, 17).Xor(rotr32(w7, 19)).Xor(w7.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[41])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w10 = w10.Add(rotr32(w11, 7).Xor(rotr32(w11, 18)).Xor(w11.ShiftAllRight(3))).Add(w3).Add(rotr32(w8, 17).Xor(rotr32(w8, 19)).Xor(w8.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[42])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w11 = w11.Add(rotr32(w12, 7).Xor(rotr32(w12, 18)).Xor(w12.ShiftAllRight(3))).Add(w4).Add(rotr32(w9, 17).Xor(rotr32(w9, 19)).Xor(w9.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[43])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w12 = w12.Add(rotr32(w13, 7).Xor(rotr32(w13, 18)).Xor(w13.ShiftAllRight(3))).Add(w5).Add(rotr32(w10, 17).Xor(rotr32(w10, 19)).Xor(w10.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[44])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w13 = w13.Add(rotr32(w14, 7).Xor(rotr32(w14, 18)).Xor(w14.ShiftAllRight(3))).Add(w6).Add(rotr32(w11, 17).Xor(rotr32(w11, 19)).Xor(w11.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[45])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w14 = w14.Add(rotr32(w15, 7).Xor(rotr32(w15, 18)).Xor(w15.ShiftAllRight(3))).Add(w7).Add(rotr32(w12, 17).Xor(rotr32(w12, 19)).Xor(w12.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[46])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w15 = w15.Add(rotr32(w0, 7).Xor(rotr32(w0, 18)).Xor(w0.ShiftAllRight(3))).Add(w8).Add(rotr32(w13, 17).Xor(rotr32(w13, 19)).Xor(w13.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[47])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w0 = w0.Add(rotr32(w1, 7).Xor(rotr32(w1, 18)).Xor(w1.ShiftAllRight(3))).Add(w9).Add(rotr32(w14, 17).Xor(rotr32(w14, 19)).Xor(w14.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[48])).Add(w0) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w1 = w1.Add(rotr32(w2, 7).Xor(rotr32(w2, 18)).Xor(w2.ShiftAllRight(3))).Add(w10).Add(rotr32(w15, 17).Xor(rotr32(w15, 19)).Xor(w15.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[49])).Add(w1) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w2 = w2.Add(rotr32(w3, 7).Xor(rotr32(w3, 18)).Xor(w3.ShiftAllRight(3))).Add(w11).Add(rotr32(w0, 17).Xor(rotr32(w0, 19)).Xor(w0.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[50])).Add(w2) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w3 = w3.Add(rotr32(w4, 7).Xor(rotr32(w4, 18)).Xor(w4.ShiftAllRight(3))).Add(w12).Add(rotr32(w1, 17).Xor(rotr32(w1, 19)).Xor(w1.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[51])).Add(w3) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w4 = w4.Add(rotr32(w5, 7).Xor(rotr32(w5, 18)).Xor(w5.ShiftAllRight(3))).Add(w13).Add(rotr32(w2, 17).Xor(rotr32(w2, 19)).Xor(w2.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[52])).Add(w4) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w5 = w5.Add(rotr32(w6, 7).Xor(rotr32(w6, 18)).Xor(w6.ShiftAllRight(3))).Add(w14).Add(rotr32(w3, 17).Xor(rotr32(w3, 19)).Xor(w3.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[53])).Add(w5) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w6 = w6.Add(rotr32(w7, 7).Xor(rotr32(w7, 18)).Xor(w7.ShiftAllRight(3))).Add(w15).Add(rotr32(w4, 17).Xor(rotr32(w4, 19)).Xor(w4.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[54])).Add(w6) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w7 = w7.Add(rotr32(w8, 7).Xor(rotr32(w8, 18)).Xor(w8.ShiftAllRight(3))).Add(w0).Add(rotr32(w5, 17).Xor(rotr32(w5, 19)).Xor(w5.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[55])).Add(w7) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + w8 = w8.Add(rotr32(w9, 7).Xor(rotr32(w9, 18)).Xor(w9.ShiftAllRight(3))).Add(w1).Add(rotr32(w6, 17).Xor(rotr32(w6, 19)).Xor(w6.ShiftAllRight(10))) + t1 = h.Add(rotr32(e, 6).Xor(rotr32(e, 11)).Xor(rotr32(e, 25))).Add(e.And(f).Xor(g.AndNot(e))).Add(archsimd.LoadUint32x16Array(&sha256K16[56])).Add(w8) + t2 = rotr32(a, 2).Xor(rotr32(a, 13)).Xor(rotr32(a, 22)).Add(a.And(b).Xor(a.Xor(b).And(c))) + d = d.Add(t1) + h = t1.Add(t2) + w9 = w9.Add(rotr32(w10, 7).Xor(rotr32(w10, 18)).Xor(w10.ShiftAllRight(3))).Add(w2).Add(rotr32(w7, 17).Xor(rotr32(w7, 19)).Xor(w7.ShiftAllRight(10))) + t1 = g.Add(rotr32(d, 6).Xor(rotr32(d, 11)).Xor(rotr32(d, 25))).Add(d.And(e).Xor(f.AndNot(d))).Add(archsimd.LoadUint32x16Array(&sha256K16[57])).Add(w9) + t2 = rotr32(h, 2).Xor(rotr32(h, 13)).Xor(rotr32(h, 22)).Add(h.And(a).Xor(h.Xor(a).And(b))) + c = c.Add(t1) + g = t1.Add(t2) + w10 = w10.Add(rotr32(w11, 7).Xor(rotr32(w11, 18)).Xor(w11.ShiftAllRight(3))).Add(w3).Add(rotr32(w8, 17).Xor(rotr32(w8, 19)).Xor(w8.ShiftAllRight(10))) + t1 = f.Add(rotr32(c, 6).Xor(rotr32(c, 11)).Xor(rotr32(c, 25))).Add(c.And(d).Xor(e.AndNot(c))).Add(archsimd.LoadUint32x16Array(&sha256K16[58])).Add(w10) + t2 = rotr32(g, 2).Xor(rotr32(g, 13)).Xor(rotr32(g, 22)).Add(g.And(h).Xor(g.Xor(h).And(a))) + b = b.Add(t1) + f = t1.Add(t2) + w11 = w11.Add(rotr32(w12, 7).Xor(rotr32(w12, 18)).Xor(w12.ShiftAllRight(3))).Add(w4).Add(rotr32(w9, 17).Xor(rotr32(w9, 19)).Xor(w9.ShiftAllRight(10))) + t1 = e.Add(rotr32(b, 6).Xor(rotr32(b, 11)).Xor(rotr32(b, 25))).Add(b.And(c).Xor(d.AndNot(b))).Add(archsimd.LoadUint32x16Array(&sha256K16[59])).Add(w11) + t2 = rotr32(f, 2).Xor(rotr32(f, 13)).Xor(rotr32(f, 22)).Add(f.And(g).Xor(f.Xor(g).And(h))) + a = a.Add(t1) + e = t1.Add(t2) + w12 = w12.Add(rotr32(w13, 7).Xor(rotr32(w13, 18)).Xor(w13.ShiftAllRight(3))).Add(w5).Add(rotr32(w10, 17).Xor(rotr32(w10, 19)).Xor(w10.ShiftAllRight(10))) + t1 = d.Add(rotr32(a, 6).Xor(rotr32(a, 11)).Xor(rotr32(a, 25))).Add(a.And(b).Xor(c.AndNot(a))).Add(archsimd.LoadUint32x16Array(&sha256K16[60])).Add(w12) + t2 = rotr32(e, 2).Xor(rotr32(e, 13)).Xor(rotr32(e, 22)).Add(e.And(f).Xor(e.Xor(f).And(g))) + h = h.Add(t1) + d = t1.Add(t2) + w13 = w13.Add(rotr32(w14, 7).Xor(rotr32(w14, 18)).Xor(w14.ShiftAllRight(3))).Add(w6).Add(rotr32(w11, 17).Xor(rotr32(w11, 19)).Xor(w11.ShiftAllRight(10))) + t1 = c.Add(rotr32(h, 6).Xor(rotr32(h, 11)).Xor(rotr32(h, 25))).Add(h.And(a).Xor(b.AndNot(h))).Add(archsimd.LoadUint32x16Array(&sha256K16[61])).Add(w13) + t2 = rotr32(d, 2).Xor(rotr32(d, 13)).Xor(rotr32(d, 22)).Add(d.And(e).Xor(d.Xor(e).And(f))) + g = g.Add(t1) + c = t1.Add(t2) + w14 = w14.Add(rotr32(w15, 7).Xor(rotr32(w15, 18)).Xor(w15.ShiftAllRight(3))).Add(w7).Add(rotr32(w12, 17).Xor(rotr32(w12, 19)).Xor(w12.ShiftAllRight(10))) + t1 = b.Add(rotr32(g, 6).Xor(rotr32(g, 11)).Xor(rotr32(g, 25))).Add(g.And(h).Xor(a.AndNot(g))).Add(archsimd.LoadUint32x16Array(&sha256K16[62])).Add(w14) + t2 = rotr32(c, 2).Xor(rotr32(c, 13)).Xor(rotr32(c, 22)).Add(c.And(d).Xor(c.Xor(d).And(e))) + f = f.Add(t1) + b = t1.Add(t2) + w15 = w15.Add(rotr32(w0, 7).Xor(rotr32(w0, 18)).Xor(w0.ShiftAllRight(3))).Add(w8).Add(rotr32(w13, 17).Xor(rotr32(w13, 19)).Xor(w13.ShiftAllRight(10))) + t1 = a.Add(rotr32(f, 6).Xor(rotr32(f, 11)).Xor(rotr32(f, 25))).Add(f.And(g).Xor(h.AndNot(f))).Add(archsimd.LoadUint32x16Array(&sha256K16[63])).Add(w15) + t2 = rotr32(b, 2).Xor(rotr32(b, 13)).Xor(rotr32(b, 22)).Add(b.And(c).Xor(b.Xor(c).And(d))) + e = e.Add(t1) + a = t1.Add(t2) + h0 = h0.Add(a) + h1 = h1.Add(b) + h2 = h2.Add(c) + h3 = h3.Add(d) + h4 = h4.Add(e) + h5 = h5.Add(f) + h6 = h6.Add(g) + h7 = h7.Add(h) + } + h0.StoreArray(&out[0]) + h1.StoreArray(&out[1]) + h2.StoreArray(&out[2]) + h3.StoreArray(&out[3]) + h4.StoreArray(&out[4]) + h5.StoreArray(&out[5]) + h6.StoreArray(&out[6]) + h7.StoreArray(&out[7]) +} diff --git a/sei-tendermint/crypto/tmhash/zeroupper_amd64.s b/sei-tendermint/crypto/tmhash/zeroupper_amd64.s new file mode 100644 index 0000000000..6c4dd6d709 --- /dev/null +++ b/sei-tendermint/crypto/tmhash/zeroupper_amd64.s @@ -0,0 +1,8 @@ +//go:build goexperiment.simd && amd64 + +#include "textflag.h" + +// func vzeroupper() +TEXT ·vzeroupper(SB), NOSPLIT, $0-0 + VZEROUPPER + RET