diff --git a/ext/Cargo.toml b/ext/Cargo.toml index 8b3919aba8..29745ae957 100644 --- a/ext/Cargo.toml +++ b/ext/Cargo.toml @@ -59,9 +59,27 @@ concurrent = [ odd-primes = ["fp/odd-primes", "algebra/odd-primes", "sseq/odd-primes"] logging = [] nassau = [] +# Dispatch large p=2 matrix products to the Hopper GPU backend via `fp`. +gpu = ["fp/gpu"] [workspace] members = [ + "crates/algebra", + "crates/bivec", + "crates/fp", + "crates/fp-cuda", + "crates/maybe-rayon", + "crates/once", + "crates/query", + "crates/sseq", +] +# `fp-cuda` builds its CUDA kernel with nvcc (falling back to a stub PTX when +# nvcc is absent) and is opt-in. It stays out of the default member set so plain +# workspace commands (`cargo build`, `cargo test`, `nix run .#test`) don't pull +# it in; `fp`'s `gpu` feature depends on it explicitly when the GPU backend is +# wanted. +default-members = [ + ".", "crates/algebra", "crates/bivec", "crates/fp", diff --git a/ext/crates/fp-cuda/Cargo.toml b/ext/crates/fp-cuda/Cargo.toml new file mode 100644 index 0000000000..14dadeb5ba --- /dev/null +++ b/ext/crates/fp-cuda/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "fp-cuda" +version = "0.1.0" +edition = "2024" +authors = ["Joey Beauvais-Feisthauer "] +description = "CUDA backend for fp's F_2 BLAS routines (Hopper wgmma.b1 via inline PTX)" +license = "MIT OR Apache-2.0" +publish = false + +# This crate ships a CUDA C++ kernel (cuda_kernels/matmul_b1.cu) that is +# compiled to PTX by nvcc at build time. The Rust side uses `cudarc` (stable +# Rust, runtime dynamic-loading of the CUDA driver) for the host driver-API +# surface: module load, device buffers, launch, and the raw `cuTensorMapEncodeTiled` +# call for the TMA descriptors. +# +# Build with: +# cargo build -p fp-cuda +# +# Prerequisites: nvcc (CUDA Toolkit 12.x+) on PATH, Hopper GPU at runtime. +# Plain `cargo build` from the workspace root skips this crate via the +# `default-members` entry in the workspace Cargo.toml. + +build = "build.rs" + +[dependencies] +# Driver API only; dynamic-loading means the crate builds with no CUDA present +# (we still need nvcc at build time to produce the PTX, and a driver at runtime). +# cuda-12080 only selects which pre-generated driver bindings to compile; with +# dynamic-loading the actual driver is dlopen'd at runtime, so no CUDA is needed +# to build and a newer driver (12.8+/13.x) still works. +cudarc = { version = "0.19", default-features = false, features = [ + "std", + "driver", + "nvrtc", + "cuda-12080", + "dynamic-loading", +] } + +[dev-dependencies] +# `fp` is a dev-dependency only: the library API is fp-agnostic (raw limbs), so +# the higher-level `fp` crate can depend on `fp-cuda` without a cycle. The +# examples/benches still use `fp::Matrix` for random generation and CPU +# cross-checks (dev-dependency cycles are permitted by Cargo). +# `concurrent` (not `cuda`) so the CPU cross-checks/baselines in the timing +# examples run multi-threaded — a fair baseline against the GPU path — while the +# device path is reached directly through fp-cuda, never through fp's dispatch. +fp = { path = "../fp", default-features = false, features = ["concurrent"] } +criterion = { version = "0.5", features = ["html_reports"] } +rand = "0.9" + +[[bench]] +name = "matmul_b1" +harness = false diff --git a/ext/crates/fp-cuda/README.md b/ext/crates/fp-cuda/README.md new file mode 100644 index 0000000000..2fca6bd2c4 --- /dev/null +++ b/ext/crates/fp-cuda/README.md @@ -0,0 +1,212 @@ +# fp-cuda + +CUDA backend for the F₂ matrix multiplication implemented in `crates/fp/src/blas/`. +The Hopper memory pipeline is used end-to-end: kernel written in CUDA C++ with +inline PTX for **TMA bulk tensor loads** with **128B swizzle** +(`cp.async.bulk.tensor.2d`), **mbarrier**-based completion sync, and the binary +tensor cores +(`wgmma.mma_async.sync.aligned.m64n64k256.row.col.s32.b1.b1.and.popc`). +Both operands are pre-arranged into plain row-major K-major tiles on the host; +the TMA applies the swizzle that the wgmma matrix descriptors expect. +Rust-side glue uses [`cudarc`](https://crates.io/crates/cudarc) for the host +driver-API surface (module load, device buffers, typed launch) and its +`driver::sys` raw bindings for the `cuTensorMapEncodeTiled` call that builds the +TMA descriptors. `cudarc` is stable Rust and dynamically loads the CUDA driver +at runtime, so the Rust side builds with no CUDA present. + +This crate is **excluded from the workspace's `default-members`**, so plain +`cargo build` / `nix run .#test` ignore it. It is opt-in: building requires +nvcc on `PATH` and (at runtime) a Hopper-class GPU. + +## Prerequisites + +1. **nvcc** (CUDA Toolkit 12.x+, since TMA + wgmma require 12.0+) on `PATH`, + with Hopper (sm_90a) support. Override the binary location with the + `NVCC` env var if needed. +2. A **Hopper GPU** at runtime (sm_90a). The kernel is built for `sm_90a`, an + architecture-specific target that is **not** forward-compatible, so the PTX + runs only on Hopper — not on pre-Hopper devices (which lack the `wgmma.*` and + `cp.async.bulk.tensor.*` instructions it emits) nor on newer architectures + such as Blackwell (`sm_100`). + +Builds on **stable** Rust — no nightly toolchain required. (`nvcc` is still +needed at build time to compile the kernel to PTX, and a CUDA driver at runtime.) + +## Building + +```bash +# From the workspace root; the leading -p selects this crate explicitly. +cargo build -p fp-cuda +``` + +`build.rs` invokes nvcc on `cuda_kernels/matmul_b1.cu` and emits +`matmul_b1.ptx` into the cargo `OUT_DIR`. `src/lib.rs` embeds it via +`include_bytes!` and loads it at runtime through cudarc. + +**When nvcc is absent** (CI, or a contributor without the CUDA Toolkit) +`build.rs` emits a *stub* PTX and prints a warning instead of failing, so the +crate stays `cargo check`/`clippy`-able everywhere — including under +`cargo check --workspace --all-features`, which the workspace CI runs on a +machine with no CUDA. The stub carries no kernel, so at runtime +`GpuContext::new` returns an error and callers (e.g. `fp`'s `cuda` dispatch) +fall back to the CPU path. An nvcc that *is* present but fails to compile is +still a hard build error. + +## Running + +```bash +# Smoke test (multiplies a few small shapes, asserts CPU↔GPU equality): +cargo run -p fp-cuda --example matmul_b1_demo + +# Benchmark against the CPU AVX-512 path in fp::blas: +cargo bench -p fp-cuda +``` + +The bench compares each square size in `{128, 256, 512, 1024, 2048, 4096, 8192}` +against `fp::blas::fast_mul_concurrent`, asserts bit-equality of the outputs, +and prints binary TOPS for both backends. + +## Why excluded from `default-members`? + +Contributors without nvcc would otherwise see this crate's build fail every +time they run `cargo build`. Keeping it out of the default member set means: + +- `cargo build`, `cargo test`, `cargo fmt`, `nix run .#test` from the + workspace root behave exactly as before. +- Tooling that wants this crate explicitly opts in with `-p fp-cuda`. + +The crate is still a workspace **member**, so `cargo metadata` sees it, +`rust-analyzer` indexes it, and shared dependency resolution works. + +## Status + +The full pipeline (host row-major pre-arrangement → TMA 128B-swizzle loads → +mbarrier sync → persistent grid + clusters/multicast → register-blocked +`m64n128k256` wgmma.b1 strips → bit-pack → double-buffered TMA bulk output +store) is **validated on an H200 NVL (sm_90, CUDA 13.0 driver / 12.4 toolkit, +2026-07-07)** and earlier on an H100 NVL. The PTX JITs at module load, the +dynamic-SMEM opt-in (~166 KB) and all three TMA descriptors are accepted, and +outputs are **bit-exact** against the CPU `fp::blas` path across `matmul_b1_demo` +(64…8192) and the kernel-only bench (4096…32768, incl. a full 32768³ CPU +cross-check). + +Throughput, **kernel-only** (host setup + H2D/D2H excluded — the comparison the +~100-TOPS pre-swizzle baseline was measured at), H200 NVL: + +| size (M=K=N) | binary TOPS | ms/launch | +|--------------|-------------|-----------| +| 4096 | ~4,000 | 0.034 | +| 8192 | ~6,700 | 0.163 | +| 16384 | ~8,600 | 1.02 | +| 32768 | ~9,600 | 7.33 | + +i.e. roughly a **~96× kernel speedup** over the ~100-TOPS pre-swizzle state, and +the >16384 L2 cliff is **gone** — throughput now *climbs* with size. + +Getting there took closing an L2-residency-of-B cliff, then a bandwidth wall, +then a latency wall (all measured on-device, since ncu can't attach through the +CUDA-13 driver — see the standalone wgmma/L2 microbenchmark approach): +- **Persistent grid + grouped rasterization + clusters/TMA-multicast** (Phases + 8–9) keep B's reuse distance short so it stays L2-resident. `bench_shapes` now + shows equal-FLOPs B-fits and B-spills shapes running at the *same* throughput — + the cliff is closed. +- **Register-blocking** (Phase 10): each CTA computes a 192×128 output block as 3 + `m64n128` strips sharing one loaded B sub-tile, cutting operand-refill + bytes/MAC by 33% — the kernel had become L2→SMEM-refill bound (~8 TB/s + sustained vs a ~12.5k-TOPS single-warpgroup wgmma ceiling). +- **Epilogue overlap** (Phase 11): double-buffered `sC` + a deferred + `cp.async.bulk.wait_group.read 1` so tile T's output store drains during tile + T+1's compute. +- **Fence hoisting** (Phase 12): one `wgmma.fence` before the K-loop instead of + two per k-chunk (a warpgroup-wide sync). After blocking, skipping an entire + operand load barely changes throughput — the kernel is now **TMA-latency + bound** (deeper pipelines help; pipeline depth is capped by SMEM at STAGES=4). + +Run `cargo run --release -p fp-cuda --example bench_shapes` (L2-residency check) +or `--example tune` (fast throughput sweep) to reproduce. + +The end-to-end `cargo bench` figures (≤30 TOPS) are dominated by host +serialization and the TMA-layout pre-arrangement; use `cargo run --release -p +fp-cuda --example bench_kernel_only` for the kernel number. + +Reproduce in this order (each gates the next): + +1. **64×256×64 identity / small product first.** Smallest path that exercises one + swizzled tile end-to-end (the first `matmul_b1_demo` case). A failure here + points at the swizzled wgmma descriptor constants (`DESC_LBO = 16`, + `DESC_SBO = 1024`, per-k256 advance of 32 bytes), or a host-layout / TMA-box + mismatch. These derive from CUTLASS `make_gmma_desc` + (`LayoutType::B128`). +2. **Full size sweep** via `cargo run -p fp-cuda --example matmul_b1_demo` + (bit-exact CPU↔GPU for 64…8192). +3. **Kernel-only throughput + correctness** via + `cargo run --release -p fp-cuda --example bench_kernel_only`; compare binary + TOPS against the ~100-TOPS pre-swizzle baseline. + +Other points worth a trace once: the dynamic-SMEM base must be 128-byte aligned +for TMA (declared `extern __shared__ __align__(128)`), and the per-stage +`expect_tx = (TILE + TILE_B) * 8` bytes (one A tile + one B tile) must match +the `cp.async.bulk.tensor.complete_tx::bytes` notifications from the two issued +TMA loads. + +## Roadmap + +Following the optimization ladder of Pranjal Shankhdhar's "Outperforming cuBLAS +on H100" worklog, adapted to the binary (`b1`) GF(2) kernel. + +Done: + +- **128B swizzle** (Phase 3) on both operands — TMA loads with + `CU_TENSOR_MAP_SWIZZLE_128B`; wgmma descriptors set `layout_type = 1` + (LBO = 16 B, SBO = 1024 B), avoiding bank conflicts. The SMEM K-tile is 1024 + bits (a full 128B K-major swizzle atom = 4 k256 sub-chunks); per-stage wgmmas + run behind one `commit_group`/`wait_group` and accumulate in-hardware + (`scale-D = 1`). Operands moved to dynamic shared memory. Host pre-arrangement + is plain row-major tiles (no `cm()` interleave). +- **Wide binary MMA** (Phase 4) — began with one `m64n256k256` per k-step + (replacing four `m64n64k256`); the current kernel uses `m64n128k256` strips + (see Phase 10). +- **Register reallocation** (Phase 5) — `setmaxnreg.dec(40)` in the producer, + `setmaxnreg.inc(N)` in the consumer (N sized to the accumulator block). +- **Deeper pipeline** (Phase 6) — `STAGES` full/empty buffers (now 4). +- **TMA output store** (Phase 7) — the packed `sC` tile is written back with a + single `cp.async.bulk.tensor.2d.global.shared::cta`; C is padded to whole + NG-limb column groups so every stored tile is complete. +- **Persistent kernel + grouped rasterization** (Phase 8) — a 1-D grid of + ~SM-count CTAs sweeps the output tiles in a grouped-along-M order (`GROUP_M` + M-tiles per band), shortening each B-panel's reuse distance to keep it + L2-resident. Cuts B's HBM re-reads by ~`GROUP_M`. +- **Clusters + TMA multicast** (Phase 9) — `CLUSTER` CTAs along M form a + thread-block cluster and share one HBM read of each B-panel via + `cp.async.bulk.tensor…multicast::cluster` (each computes a different M-tile). + Cluster-wide empty barrier; the pipeline barriers init once and flow + continuously across tiles. Mirrors the proven `pranjalssh/fast.cu` matmul_9. +- **Register-blocking** (Phase 10) — each CTA computes a `TM×NB` output block + (`MSTRIPS` `m64n128` strips × `NB`=128 cols) whose strips all reuse one loaded + B sub-tile, cutting operand-refill bytes/MAC (the bottleneck after Phase 9). + `MSTRIPS`=3 → a 192×128 block, −33% bytes/MAC. `MSTRIPS`/`TILE_M`/`NG` in the + kernel and `src/lib.rs` must match. +- **Epilogue overlap** (Phase 11) — double-buffered `sC` + deferred + `cp.async.bulk.wait_group.read 1` so tile T's output store overlaps tile T+1's + compute. +- **Fence hoisting** (Phase 12) — one `wgmma.fence` before the K-loop instead of + two per k-chunk. + +Phases 8–9 were validated on hardware (H200 NVL) alongside 10–12. + +- **Wired into `fp`** — the `fp` crate has an optional `gpu` feature + (`cargo …--features fp/gpu`) that pulls in `fp-cuda` and dispatches large + `p = 2` products from `<&Matrix as Mul>::mul` to the GPU, falling back to the + CPU BLAS kernel when no device is present, the size is below + `FP_CUDA_THRESHOLD` (default 2048), or a launch fails. `fp-cuda`'s library API + is fp-agnostic (raw row-major limb slices) so the dependency is acyclic; the + `Matrix` glue lives on the `fp` side (`src/blas/cuda.rs`) and in the + examples/benches (a dev-dependency on `fp`). `FP_CUDA_DISABLE` forces the CPU + path. See the `gpu_dispatch_matches_cpu` integration test. + +Remaining (now TMA-latency bound — the consumer out-runs per-tile TMA latency and +pipeline depth is SMEM-capped at `STAGES`=4): + +- Keep operands resident on the device across `step_resolution`'s successive + multiplications (the current dispatch re-marshals and re-copies each product). +- Extend the parent Nix flake to provide nvcc when the user opts in. diff --git a/ext/crates/fp-cuda/benches/matmul_b1.rs b/ext/crates/fp-cuda/benches/matmul_b1.rs new file mode 100644 index 0000000000..26af1f456f --- /dev/null +++ b/ext/crates/fp-cuda/benches/matmul_b1.rs @@ -0,0 +1,110 @@ +use std::time::Instant; + +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +#[path = "../examples/common/mod.rs"] +mod common; +use common::matmul_b1; +use rand::Rng; + +const SIZES: &[usize] = &[128, 256, 512, 1024, 2048, 4096, 8192]; + +fn random_matrix(rows: usize, cols: usize) -> Matrix { + let mut rng = rand::rng(); + let data_len = rows * cols.div_ceil(64); + let data: Vec = (0..data_len).map(|_| rng.random()).collect(); + Matrix::from_data(TWO, rows, cols, data) +} + +fn assert_bit_equal(cpu: &Matrix, gpu: &Matrix) { + assert_eq!(cpu.rows(), gpu.rows(), "row count mismatch"); + assert_eq!(cpu.columns(), gpu.columns(), "column count mismatch"); + assert!( + cpu == gpu, + "CPU and GPU F_2 matmul results disagree at {}x{}", + cpu.rows(), + cpu.columns(), + ); +} + +fn binary_tops(m: usize, k: usize, n: usize, secs: f64) -> f64 { + // 2 * M * N * K binary ops (one AND + one XOR per inner-product step). + 2.0 * (m as f64) * (n as f64) * (k as f64) / secs / 1e12 +} + +fn bench_square(c: &mut Criterion, gpu: &GpuContext, size: usize) { + let mut group = c.benchmark_group(format!("matmul_b1_{size}x{size}")); + group.throughput(criterion::Throughput::Elements( + (2 * size * size * size) as u64, + )); + + // One-shot correctness check (outside the criterion timing loop) before benching. + let (a, b) = (random_matrix(size, size), random_matrix(size, size)); + let cpu_ref = &a * &b; + let gpu_ref = matmul_b1(gpu, &a, &b).expect("GPU matmul launch failed"); + assert_bit_equal(&cpu_ref, &gpu_ref); + + group.bench_function("cpu_fast_mul_concurrent", |bencher| { + bencher.iter_batched( + || (random_matrix(size, size), random_matrix(size, size)), + |(a, b)| &a * &b, + BatchSize::SmallInput, + ); + }); + + group.bench_function("gpu_matmul_b1", |bencher| { + bencher.iter_batched( + || (random_matrix(size, size), random_matrix(size, size)), + |(a, b)| matmul_b1(gpu, &a, &b).expect("GPU matmul launch failed"), + BatchSize::SmallInput, + ); + }); + + group.finish(); + + // Coarse manual TFLOPS report (criterion has its own throughput, but explicit + // logging makes binary-op throughput easy to grep from the bench output). + let runs = 5; + let start = Instant::now(); + for _ in 0..runs { + let _ = matmul_b1(gpu, &a, &b).expect("GPU matmul launch failed"); + } + let gpu_avg = start.elapsed().as_secs_f64() / runs as f64; + + let start = Instant::now(); + for _ in 0..runs { + let _ = &a * &b; + } + let cpu_avg = start.elapsed().as_secs_f64() / runs as f64; + + println!( + "[matmul_b1_{size}x{size}] CPU {:.2} TOPS ({:.2} ms) GPU {:.2} TOPS ({:.2} ms) speedup \ + {:.2}x", + binary_tops(size, size, size, cpu_avg), + cpu_avg * 1e3, + binary_tops(size, size, size, gpu_avg), + gpu_avg * 1e3, + cpu_avg / gpu_avg, + ); +} + +fn bench_all(c: &mut Criterion) { + let gpu = GpuContext::new(0).expect("failed to initialise GpuContext"); + let (major, minor) = gpu + .compute_capability() + .expect("failed to query compute capability"); + println!("fp-cuda bench running on sm_{major}{minor}"); + + for &size in SIZES { + bench_square(c, &gpu, size); + } +} + +criterion_group! { + name = matmul_b1_bench; + config = Criterion::default().measurement_time(std::time::Duration::from_secs(3)); + targets = bench_all +} +criterion_main!(matmul_b1_bench); diff --git a/ext/crates/fp-cuda/build.rs b/ext/crates/fp-cuda/build.rs new file mode 100644 index 0000000000..3f6817247e --- /dev/null +++ b/ext/crates/fp-cuda/build.rs @@ -0,0 +1,76 @@ +//! Compile the CUDA C++ kernel to PTX via nvcc. +//! +//! The emitted `matmul_b1.ptx` is picked up by `src/lib.rs` via +//! `include_bytes!(concat!(env!("OUT_DIR"), "/matmul_b1.ptx"))`. +//! +//! When `nvcc` is **not on PATH** (e.g. CI, or a contributor without the CUDA +//! Toolkit), we emit a *stub* PTX instead of failing the build. This keeps the +//! crate `cargo check`/`clippy`-able everywhere — important because the `fp` +//! crate's optional `cuda` feature pulls `fp-cuda` in, and CI runs +//! `cargo check --workspace --all-features` on a machine with no CUDA. The stub +//! carries no kernel, so at runtime `GpuContext::new` fails cleanly and callers +//! fall back to the CPU path. A `nvcc` that *is* present but fails to compile is +//! still a hard error (a real kernel bug we want to surface). + +use std::{env, fs, path::PathBuf, process::Command}; + +const KERNEL_SRC: &str = "cuda_kernels/matmul_b1.cu"; +const PTX_NAME: &str = "matmul_b1.ptx"; +const ARCH: &str = "sm_90a"; + +/// A minimal, valid PTX module with no kernel. `include_bytes!` needs a file to +/// exist; loading this at runtime fails at `load_function("matmul_b1_kernel")`, +/// which the Rust side surfaces as an error so the caller uses the CPU path. +const STUB_PTX: &str = "//\n// fp-cuda stub: nvcc was unavailable at build time; no GPU kernel \ + was\n// compiled. Install the CUDA Toolkit (12.x+) and rebuild to enable \ + the GPU\n// backend.\n//\n.version 7.8\n.target sm_90a\n.address_size 64\n"; + +fn main() { + println!("cargo:rerun-if-changed={KERNEL_SRC}"); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=NVCC"); + + let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR not set by cargo")); + let ptx_out = out_dir.join(PTX_NAME); + + let nvcc = env::var("NVCC").unwrap_or_else(|_| "nvcc".to_string()); + + let status = Command::new(&nvcc) + .args([ + "-ptx", + "-O3", + "-std=c++17", + "--use_fast_math", + &format!("-arch={ARCH}"), + KERNEL_SRC, + "-o", + ]) + .arg(&ptx_out) + .status(); + + match status { + // nvcc ran and compiled the kernel: the real PTX is in place. + Ok(s) if s.success() => {} + // nvcc exists but failed to compile: surface it (real kernel error). + Ok(s) => panic!( + "nvcc failed to compile {KERNEL_SRC} (exit status: {s}).\nCheck that your CUDA \ + Toolkit supports {ARCH} (Hopper sm_90a)." + ), + // nvcc genuinely absent: degrade to a stub so the crate still builds. + // The GPU backend is simply unavailable at runtime. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + println!( + "cargo:warning=fp-cuda: nvcc not found ('{nvcc}': {e}); emitting a stub PTX. The \ + GPU backend will be unavailable at runtime. Install the CUDA Toolkit (12.x+) and \ + rebuild to enable it." + ); + fs::write(&ptx_out, STUB_PTX).expect("failed to write stub PTX to OUT_DIR"); + } + // nvcc is present but couldn't be executed (permissions, a broken exec, + // etc.): fail fast rather than silently hiding a broken CUDA setup. + Err(e) => panic!( + "failed to run nvcc ('{nvcc}': {e}). Set the NVCC env var to a working nvcc, or unset \ + it to fall back to a stub only when nvcc is absent." + ), + } +} diff --git a/ext/crates/fp-cuda/cuda_kernels/matmul_b1.cu b/ext/crates/fp-cuda/cuda_kernels/matmul_b1.cu new file mode 100644 index 0000000000..7dd9334b54 --- /dev/null +++ b/ext/crates/fp-cuda/cuda_kernels/matmul_b1.cu @@ -0,0 +1,1015 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Hopper wgmma.b1 F_2 GEMM kernel — 128B-swizzle operands, pipelined wgmmas, +// the widest binary MMA shape (m64n256k256), a persistent grid with grouped +// tile rasterization (Phase 8), and thread-block clusters + TMA B-multicast +// (Phase 9) — both target L2 residency of B at large N. +// +// Both operands are K-major. They are pre-arranged on the host as plain +// row-major tiles and loaded via TMA cp.async.bulk.tensor.2d with +// CU_TENSOR_MAP_SWIZZLE_128B: the TMA hardware applies the 128B swizzle on the +// way into SMEM, landing the data exactly where the swizzled wgmma matrix +// descriptor expects it — so the host emits the natural layout and there is no +// hand-rolled interleave. +// +// Each loaded tile spans a full 128B K-major swizzle atom (8 rows × 1024 bits), +// i.e. KSUB = 4 consecutive k256 sub-chunks. A CTA computes a register-blocked +// TM×NB output block (MSTRIPS m64 row-strips × 128 columns). Each k256 step +// loads B once and issues MSTRIPS m64n128k256 wgmmas that all reuse it — one +// L2→SMEM read of B feeds every strip, which cuts the operand-refill bytes per +// MAC (the measured bottleneck on Hopper: the tensor core out-runs L2→SMEM +// bandwidth). Each strip accumulates into its own resident 64-reg accumulator +// (scale-D = 1, popcounts summed in-hardware), all live across the whole K loop. +// +// The grid is persistent: ~SM-count CTAs (in clusters of CLUSTER along M) sweep +// the output tile grid in a grouped-along-M rasterized order so each B-panel's +// reuse distance stays short (L2-resident). Within a cluster the CLUSTER CTAs +// share one HBM read of each B-panel via TMA multicast — each computes a +// different M-tile but receives the same B into its own SMEM. The pipeline +// barriers are initialized once and flow continuously across tiles; the empty +// barrier is cluster-wide. This mirrors the proven pattern in +// pranjalssh/fast.cu matmul_9.cuh. + +#include +#include +#include + +// ── Helpers ───────────────────────────────────────────────────────────────── + +// Build a wgmma SMEM matrix descriptor. +// p : SMEM address of the operand sub-tile (already swizzled by TMA). +// lead : leading-dimension byte offset (LBO), per CUTLASS make_gmma_desc. +// stride: stride-dimension byte offset (SBO). +// swiz : layout_type — 0 = none, 1 = 128B, 2 = 64B, 3 = 32B. +// Byte offsets are stored with their low 4 bits dropped (uint128 units). +__device__ __forceinline__ uint64_t make_desc( + const void* p, uint32_t lead, uint32_t stride, uint32_t swiz) { + uint32_t a = (uint32_t)__cvta_generic_to_shared(p); + uint64_t d = 0; + d |= ((uint64_t)a >> 4) & 0x3FFFULL; + d |= ((uint64_t)(lead >> 4) & 0x3FFFULL) << 16; + d |= ((uint64_t)(stride >> 4) & 0x3FFFULL) << 32; + d |= ((uint64_t)(swiz & 0x3)) << 62; + return d; +} + +__device__ __forceinline__ void mbar_init(uint64_t* b, uint32_t cnt) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" + :: "r"((uint32_t)__cvta_generic_to_shared(b)), "r"(cnt)); +} +__device__ __forceinline__ void mbar_tx(uint64_t* b, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" + :: "r"((uint32_t)__cvta_generic_to_shared(b)), "r"(bytes) : "memory"); +} +__device__ __forceinline__ void mbar_wait(uint64_t* b, uint32_t phase) { + uint32_t a = (uint32_t)__cvta_generic_to_shared(b); + asm volatile( + "{ .reg .pred p;\n" + " L: mbarrier.try_wait.parity.shared::cta.b64 p, [%0], %1;\n" + " @!p bra L;\n" + "}\n" :: "r"(a), "r"(phase) : "memory"); +} +__device__ __forceinline__ void tma_2d( + void* dst, const CUtensorMap* tm, int x, int y, uint64_t* b) { + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes" + " [%0], [%1, {%2,%3}], [%4];\n" + :: "r"((uint32_t)__cvta_generic_to_shared(dst)), + "l"((uint64_t)tm), "r"(x), "r"(y), + "r"((uint32_t)__cvta_generic_to_shared(b)) + : "memory"); +} + +// ── Cluster helpers (Phase 9: clusters + TMA multicast) ─────────────────────── +// All mirror the proven pattern in pranjalssh/fast.cu matmul_9.cuh. + +// This CTA's rank within its cluster (0..CLUSTER-1). +__device__ __forceinline__ uint32_t cluster_ctarank() { + uint32_t r; + asm volatile("mov.u32 %0, %cluster_ctarank;\n" : "=r"(r) :); + return r; +} + +// Cluster-wide barrier: every thread of every CTA in the cluster must arrive. +__device__ __forceinline__ void cluster_sync() { + asm volatile("barrier.cluster.arrive;\n" ::: "memory"); + asm volatile("barrier.cluster.wait;\n" ::: "memory"); +} + +// Arrive (count 1) on the mbarrier `b` located in cluster-mate CTA `cta_id`, +// using mapa to translate the local SMEM address into that CTA's window. +__device__ __forceinline__ void arrive_cluster(uint64_t* b, uint32_t cta_id) { + uint32_t local = (uint32_t)__cvta_generic_to_shared(b); + asm volatile( + "{ .reg .b32 rem;\n" + " mapa.shared::cluster.u32 rem, %0, %1;\n" + " mbarrier.arrive.shared::cluster.b64 _, [rem], 1;\n" + "}\n" :: "r"(local), "r"(cta_id) : "memory"); +} + +// TMA load with cluster multicast: one HBM read of the source tile is fanned +// out into the SMEM of every CTA whose bit is set in `mask` (same `dst` SMEM +// offset and `b` mbarrier offset in each), and counts complete_tx bytes against +// each of their barriers. Issued by a single thread of one CTA. +__device__ __forceinline__ void tma_2d_multicast( + void* dst, const CUtensorMap* tm, int x, int y, uint64_t* b, uint16_t mask) { + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global" + ".mbarrier::complete_tx::bytes.multicast::cluster" + " [%0], [%1, {%2,%3}], [%4], %5;\n" + :: "r"((uint32_t)__cvta_generic_to_shared(dst)), + "l"((uint64_t)tm), "r"(x), "r"(y), + "r"((uint32_t)__cvta_generic_to_shared(b)), "h"(mask) + : "memory"); +} + +// m64n128k256 binary MMA, scale-D = 1 (accumulate into the 64 s32 regs of +// `acc`). da/db are the swizzled operand descriptors. Half the N of the +// widest binary shape, so MSTRIPS of these share one B tile to cut refill BW. +__device__ __forceinline__ void wgmma_n128(int32_t acc[64], uint64_t da, uint64_t db) { + asm volatile( + "wgmma.mma_async.sync.aligned.m64n128k256.row.col.s32.b1.b1.and.popc " + "{" \ + "%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15," \ + "%16,%17,%18,%19,%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31," \ + "%32,%33,%34,%35,%36,%37,%38,%39,%40,%41,%42,%43,%44,%45,%46,%47," \ + "%48,%49,%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,%60,%61,%62,%63}," \ + "%64,%65, 1;\n" + : "+r"(acc[0]),"+r"(acc[1]),"+r"(acc[2]),"+r"(acc[3]),"+r"(acc[4]),"+r"(acc[5]),"+r"(acc[6]),"+r"(acc[7]), + "+r"(acc[8]),"+r"(acc[9]),"+r"(acc[10]),"+r"(acc[11]),"+r"(acc[12]),"+r"(acc[13]),"+r"(acc[14]),"+r"(acc[15]), + "+r"(acc[16]),"+r"(acc[17]),"+r"(acc[18]),"+r"(acc[19]),"+r"(acc[20]),"+r"(acc[21]),"+r"(acc[22]),"+r"(acc[23]), + "+r"(acc[24]),"+r"(acc[25]),"+r"(acc[26]),"+r"(acc[27]),"+r"(acc[28]),"+r"(acc[29]),"+r"(acc[30]),"+r"(acc[31]), + "+r"(acc[32]),"+r"(acc[33]),"+r"(acc[34]),"+r"(acc[35]),"+r"(acc[36]),"+r"(acc[37]),"+r"(acc[38]),"+r"(acc[39]), + "+r"(acc[40]),"+r"(acc[41]),"+r"(acc[42]),"+r"(acc[43]),"+r"(acc[44]),"+r"(acc[45]),"+r"(acc[46]),"+r"(acc[47]), + "+r"(acc[48]),"+r"(acc[49]),"+r"(acc[50]),"+r"(acc[51]),"+r"(acc[52]),"+r"(acc[53]),"+r"(acc[54]),"+r"(acc[55]), + "+r"(acc[56]),"+r"(acc[57]),"+r"(acc[58]),"+r"(acc[59]),"+r"(acc[60]),"+r"(acc[61]),"+r"(acc[62]),"+r"(acc[63]) + : "l"(da), "l"(db)); +} +__device__ __forceinline__ void wgmma_fence() { asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory"); } +__device__ __forceinline__ void wgmma_commit() { asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory"); } +__device__ __forceinline__ void wgmma_wait() { asm volatile("wgmma.wait_group.sync.aligned 0;\n" ::: "memory"); } + +// TMA bulk tensor store (SMEM → global) plus its completion group helpers and +// the async-proxy fence that makes generic-proxy SMEM writes visible to it. +__device__ __forceinline__ void tma_store_2d( + const CUtensorMap* tm, int x, int y, const void* src) { + asm volatile( + "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [%0, {%1, %2}], [%3];\n" + :: "l"((uint64_t)tm), "r"(x), "r"(y), + "r"((uint32_t)__cvta_generic_to_shared(src)) : "memory"); +} +__device__ __forceinline__ void tma_store_commit() { asm volatile("cp.async.bulk.commit_group;\n" ::: "memory"); } +__device__ __forceinline__ void tma_store_wait() { asm volatile("cp.async.bulk.wait_group 0;\n" ::: "memory"); } +// Wait until all but the most-recent store group have *read* their SMEM source +// (`.read` = don't wait for global visibility). Lets a double-buffered sC free +// the buffer from two tiles ago while the newest store drains in the background. +__device__ __forceinline__ void tma_store_wait_read1() { asm volatile("cp.async.bulk.wait_group.read 1;\n" ::: "memory"); } +__device__ __forceinline__ void fence_async_shared(){ asm volatile("fence.proxy.async.shared::cta;\n" ::: "memory"); } + +// Per-warpgroup register reallocation (warpgroup-aligned). The producer needs +// few registers, so it releases its surplus; the consumer (MSTRIPS*ACC_N-reg +// accumulator) claims them. Counts must be multiples of 8 in [24,256]; at +// 1 CTA/SM the SM's 64K-register budget is ample (128*(40+232) = 34816 at +// MSTRIPS=3), so the binding limit is the 255-reg-per-thread hardware cap. +#define SET_MAXNREG_DEC(N) asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n" :: "n"(N)) +#define SET_MAXNREG_INC(N) asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n" :: "n"(N)) + +// Output block = MSTRIPS m64 row-strips × NB columns per CTA. Each k256 step +// issues MSTRIPS m64n128 wgmmas that SHARE one B sub-tile, so a single L2→SMEM +// load of B feeds MSTRIPS strips — cutting refill bytes/MAC (the bottleneck) by +// ~1/(1+NB/BM). MSTRIPS is the block knob: 2 → 128×128 block (−20% bytes/MAC, +// 128 acc regs), 3 → 192×128 (−33%, 192 acc regs). NB is fixed at 128 (the +// wgmma_n128 shape); acc regs per thread = MSTRIPS*ACC_N ≤ 240. +constexpr int MSTRIPS = 3; // m64 row-strips per CTA (block knob) +constexpr int MW = 64; // wgmma M extent (fixed for binary wgmma) +constexpr int TK = 1024, KL = TK/64; +constexpr int TM = MW*MSTRIPS; // output rows per CTA (192 at MSTRIPS=3) +constexpr int NB = 128; // n128 output width (columns) per CTA +constexpr int NG = NB/64; // 2 output column-limbs per CTA +constexpr int ACC_N = NB/2; // 64 s32 accumulator regs per m64n128 strip +constexpr int TILE_A = TM*KL; // A tile: TM rows × 16 u64 (192 rows → 3072 u64 = 24 KB) +constexpr int TILE_B = NB*KL; // B tile: 128 cols × 16 u64 = 2048 u64 = 16 KB +constexpr int STROW = MW*KL; // u64 per m64 strip in sA (64*16 = 1024 = 8 KB) +constexpr int SC_STRIDE = NG*TM; // u64 per sC output buffer (double-buffered) +constexpr int KSUB = TK/256; // 4 k256 wgmma sub-chunks per loaded tile +constexpr int KSUB_U64 = 256/64; // 4 u64 = 32 bytes per k256 sub-chunk +// K-loop pipeline depth. Cap is 4 under CLUSTER>1: each stage is 40 KB, so +// STAGES=5 needs ~206 KB, which is under the 227 KB opt-in cap BUT a cluster +// launch reserves extra shared memory (distributed-SMEM / cluster-barrier +// bookkeeping), so 206 KB intermittently over-commits and the kernel faults +// with a flaky "unspecified launch failure" (verified 2026-07-07 H200; the +// sanitizers miss it because it is an async/resource fault). STAGES=5 is stable +// only with CLUSTER=1, and gives no large-N speedup there, so 4 it is. +constexpr int STAGES = 4; // K-loop pipeline depth (full/empty buffers) +constexpr int THREADS_PER_WG = 128; +constexpr int GROUP_M = 16; // M-tiles per rasterization group (L2 reuse knob) +constexpr int CLUSTER = 2; // CTAs per cluster along M (multicast B; reuse knob) +constexpr int PRODUCER_REGS = 40; +// Consumer holds MSTRIPS*ACC_N accumulator regs live across K, plus addressing; +// round up to a multiple of 8, ≤ 240. 1 CTA/SM so the SM reg budget is ample. +constexpr int CONSUMER_REGS = ((MSTRIPS*ACC_N + 40 + 7)/8)*8; + +// wgmma 128B K-major descriptor constants (CUTLASS make_gmma_desc, +// LayoutType::B128): LBO = 1 uint128 = 16 bytes, SBO = 8-row-brick stride = +// 1024 bytes (independent of the MN extent), swizzle = 1. A k256 sub-chunk c +// sits at byte offset c*32 within the tile (advance start_address; the +// hardware re-applies the swizzle). +constexpr uint32_t DESC_LBO = 16; +constexpr uint32_t DESC_SBO = 1024; +constexpr uint32_t DESC_SWIZ = 1; + +// ── Kernel ────────────────────────────────────────────────────────────────── + +// Producer-consumer kernel: 2 warpgroups (256 threads/CTA). +// Warpgroup 0 (t in [0, 128)) = PRODUCER: issues TMA loads in a tight +// K-loop into a STAGES-deep circular SMEM +// buffer. +// Warpgroup 1 (t in [128, 256)) = CONSUMER: waits for each stage to be full, +// runs KSUB*MSTRIPS pipelined m64n128 wgmmas +// against it, signals the stage empty so the +// producer can refill. +// +// Dynamic SMEM per CTA (carved from `smem`, 128B-aligned for TMA): +// sA[STAGES][TILE_A] = STAGES * 24576 B (TM=192 rows) +// sB[STAGES][TILE_B] = STAGES * 16384 B (NB=128 cols) +// sC[2][TM][NG] = 2 * TM * NG * 8 B (double-buffered so the output store +// of tile T overlaps tile T+1's compute) +// mbar_full[STAGES] + mbar_empty[STAGES] +// +// Per stage = sA (24 KB) + sB (16 KB) = 40 KB; STAGES=4 ≈ 163 KB total (requires +// the opt-in CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES set host-side). +// At 40 KB/stage the block runs 1 CTA/SM; STAGES is the K-pipeline-depth knob. +// +// The output block (TM rows × NG limbs) is packed row-major into sC and written +// back with a single TMA bulk store (S2G). C is padded to whole NG-limb column +// groups on the host so every stored tile is complete. +extern "C" __global__ void __cluster_dims__(CLUSTER, 1, 1) matmul_b1_kernel( + const __grid_constant__ CUtensorMap tma_a, + const __grid_constant__ CUtensorMap tma_b, + const __grid_constant__ CUtensorMap tma_c, + uint32_t m_tiles, + uint32_t n_groups, + uint32_t M, uint32_t K) +{ + extern __shared__ __align__(128) uint64_t smem[]; + uint64_t* sA = smem; // [STAGES][TILE_A] + uint64_t* sB = sA + STAGES * TILE_A; // [STAGES][TILE_B] + uint64_t* sC = sB + STAGES * TILE_B; // [2][TM][NG] row-major (double-buffered) + uint64_t* mbar_full = sC + 2 * SC_STRIDE; // [STAGES] + uint64_t* mbar_empty = mbar_full + STAGES; // [STAGES] + + const int t = threadIdx.x; + const int wg = t / THREADS_PER_WG; // 0 = producer, 1 = consumer + const int t_wg = t - wg * THREADS_PER_WG; // 0..127 within warpgroup + + const int nchunks = (K + TK - 1) / TK; + // One full A tile + one full B tile per stage (B is zero-padded on the + // host to a multiple of NB columns, so it is always a complete tile). A is + // loaded per-CTA; B arrives via multicast — both target this CTA's full + // barrier, so the expected bytes are the same as the single-CTA case. + const uint32_t expected_tx = (uint32_t)((TILE_A + TILE_B) * sizeof(uint64_t)); + + // Cluster geometry: CLUSTER CTAs along M share one B-panel via multicast, + // so the schedule walks "M-super-rows" of CLUSTER M-tiles. The host pads + // m_tiles to a multiple of CLUSTER, so m_super divides exactly. + const uint32_t rank = cluster_ctarank(); // 0..CLUSTER-1 (= M offset) + const uint32_t cluster_id = blockIdx.x / CLUSTER; + const uint32_t num_clusters = gridDim.x / CLUSTER; + const uint32_t m_super = m_tiles / CLUSTER; + const uint32_t total_cl = m_super * n_groups; + const uint16_t bmask = (uint16_t)((1u << CLUSTER) - 1u); // all ranks + + // Register reallocation is a one-time per-warpgroup action. + if (wg == 0) SET_MAXNREG_DEC(PRODUCER_REGS); + else SET_MAXNREG_INC(CONSUMER_REGS); + + // Initialize the pipeline barriers ONCE; they flow continuously across the + // persistent tile loop (no per-tile re-init, which would race with the + // cross-CTA arrivals/multicast of a cluster). The empty barrier is + // cluster-wide: it needs one arrival from every CTA's consumer. + if (t == 0) { + #pragma unroll + for (int s = 0; s < STAGES; ++s) { + mbar_init(&mbar_full[s], 1); + mbar_init(&mbar_empty[s], CLUSTER); + } + } + __syncthreads(); + cluster_sync(); // all CTAs' barriers initialized before any cross-CTA arrive + + // Pre-arrive every empty barrier cluster-wide so the producer's first + // STAGES `mbar_wait(empty, 0)` succeed immediately (stages logically free). + if (wg == 1 && t_wg < CLUSTER) { + #pragma unroll + for (int s = 0; s < STAGES; ++s) arrive_cluster(&mbar_empty[s], t_wg); + } + + // ===================== PERSISTENT CLUSTER LOOP ===================== + // A 1-D grid of clusters sweeps the M-super × N tile grid. The grouped + // rasterizer (super-row varies fastest within a GROUP_M band) keeps each + // B-panel's reuse distance short for L2 residency; the cluster additionally + // shares each B-panel HBM read across its CLUSTER CTAs via multicast. + // qidx/p are the running pipeline slot/phase, carried across tiles. + // sC is double-buffered so tile T's output store overlaps tile T+1's + // compute: cbuf ping-pongs per tile, and the store's wait is deferred one + // tile (see epilogue). titer counts this CTA's tiles for the ping-pong. + uint32_t qidx = 0, p = 0, titer = 0; + for (uint32_t ct = cluster_id; ct < total_cl; ct += num_clusters, ++titer) { + const uint32_t gid = ct / (GROUP_M * n_groups); + const uint32_t firstm = gid * GROUP_M; + const uint32_t curm = min((uint32_t)GROUP_M, m_super - firstm); + const uint32_t local = ct - gid * GROUP_M * n_groups; + const uint32_t sbi = firstm + local % curm; + const int bj = (int)(local / curm); + const int bi = (int)(sbi * CLUSTER + rank); // this CTA's M-tile + const int row0 = bi * TM, col0 = bj * NG; + uint64_t* sCb = sC + (titer & 1) * SC_STRIDE; // this tile's sC buffer + + // Before reusing this buffer, make sure the store that last used it (two + // tiles ago) has finished reading it. The deferred .read-1 wait below + // already drained it during the previous tile; this syncs all consumer + // threads to that wait before they overwrite the buffer. + if (wg == 1) { + for (int r = t_wg; r < TM; r += THREADS_PER_WG) { + #pragma unroll + for (int g = 0; g < NG; ++g) sCb[r * NG + g] = 0; + } + } + __syncthreads(); + + if (wg == 0) { + // ===================== PRODUCER ===================== + for (int kk = 0; kk < nchunks; ++kk) { + const uint32_t s = qidx; + + if (t_wg == 0) { + // Wait for all CTAs' consumers to release this stage, then + // set expected bytes (A + multicast B) and issue the loads. + mbar_wait(&mbar_empty[s], p); + mbar_tx(&mbar_full[s], expected_tx); + // A: this CTA's own TM-row block (MSTRIPS m64 strips). + tma_2d(&sA[s * TILE_A], &tma_a, 0, + (kk * m_tiles + bi) * TM, &mbar_full[s]); + // B: one HBM read, multicast into every cluster member's sB + // and counted against every member's full barrier. Issued by + // rank 0 only (its mask bit is set, so it fills itself too). + if (rank == 0) { + tma_2d_multicast(&sB[s * TILE_B], &tma_b, 0, + (kk * n_groups + bj) * NB, &mbar_full[s], + bmask); + } + } + if (++qidx == STAGES) { qidx = 0; p ^= 1; } + } + } else { + // ===================== CONSUMER ===================== + // MSTRIPS m64n128 accumulators (MSTRIPS*ACC_N s32 regs/thread), all + // resident across the whole K loop, re-zeroed per output tile. + int32_t acc[MSTRIPS][ACC_N]; + #pragma unroll + for (int si = 0; si < MSTRIPS; ++si) + #pragma unroll + for (int r = 0; r < ACC_N; ++r) acc[si][r] = 0; + + // One wgmma.fence for the whole K-loop: it orders the non-wgmma + // accumulator zeroing above before the first wgmma. Inside the loop + // the accumulators are written only by wgmma, so no per-chunk fence + // is needed (a warpgroup-wide sync we don't want 32× per tile). The + // per-chunk wgmma.wait_group 0 makes the results readable at the end. + wgmma_fence(); + for (int kk = 0; kk < nchunks; ++kk) { + const uint32_t s = qidx; + + // Wait for the producer's TMAs to finish populating this stage. + mbar_wait(&mbar_full[s], p); + + // Issue every k256 wgmma for this stage behind one commit/wait so + // they pipeline. Each k256 loads B once and reuses it across all + // MSTRIPS strips (independent accumulators → they can overlap). + // scale-D = 1 accumulates each sub-chunk in-hardware. + #pragma unroll + for (int c = 0; c < KSUB; ++c) { + uint64_t db = make_desc(&sB[s * TILE_B + c * KSUB_U64], + DESC_LBO, DESC_SBO, DESC_SWIZ); + #pragma unroll + for (int si = 0; si < MSTRIPS; ++si) { + uint64_t da = make_desc(&sA[s * TILE_A + si * STROW + c * KSUB_U64], + DESC_LBO, DESC_SBO, DESC_SWIZ); + wgmma_n128(acc[si], da, db); + } + } + wgmma_commit(); + wgmma_wait(); + + // Release this stage cluster-wide: arrive on every CTA's empty + // barrier (so rank 0 may overwrite their multicast sB). + if (t_wg < CLUSTER) arrive_cluster(&mbar_empty[s], t_wg); + if (++qidx == STAGES) { qidx = 0; p ^= 1; } + } + + // Pack each strip's NB-wide accumulator into sC's NG output limbs. + // The m64n128 fragment is the m64n64 layout tiled along N: register + // group gi (0..NB/8-1) covers output columns [gi*8, gi*8+8); within it + // this thread owns columns cb, cb+1 for rows rb and rb+8. Strip si adds + // si*MW to the row. Column c maps to limb c/64, bit c%64. + const int wid = t_wg >> 5, lane = t_wg & 31; + const int rb = wid*16 + (lane>>2), cb = (lane&3)*2; + #pragma unroll + for (int si = 0; si < MSTRIPS; ++si) { + uint64_t lo[NG] = {0}, hi[NG] = {0}; + #pragma unroll + for (int gi = 0; gi < NB/8; ++gi) { + int c0 = cb + gi*8, c1 = c0 + 1; + int l0 = c0 >> 6, b0p = c0 & 63; + int l1 = c1 >> 6, b1p = c1 & 63; + lo[l0] |= (uint64_t)(acc[si][gi*4+0]&1) << b0p; + lo[l1] |= (uint64_t)(acc[si][gi*4+1]&1) << b1p; + hi[l0] |= (uint64_t)(acc[si][gi*4+2]&1) << b0p; + hi[l1] |= (uint64_t)(acc[si][gi*4+3]&1) << b1p; + } + // Row-major sC[row*NG + limb]; padded limbs (out-of-range columns) + // get zero popcounts from the zero-padded B, so they store harmless + // zeros into C's padded region (trimmed on the host). + const int rlo = si*MW + rb, rhi = si*MW + rb + 8; + #pragma unroll + for (int g = 0; g < NG; ++g) { + uint32_t* clo = reinterpret_cast(&sCb[rlo * NG + g]); + uint32_t* chi = reinterpret_cast(&sCb[rhi * NG + g]); + atomicXor(&clo[0], (uint32_t)lo[g]); + atomicXor(&clo[1], (uint32_t)(lo[g]>>32)); + atomicXor(&chi[0], (uint32_t)hi[g]); + atomicXor(&chi[1], (uint32_t)(hi[g]>>32)); + } + } + } + + // Write the TM×NG output block back with a single TMA bulk store, then + // DEFER its wait: `.read 1` blocks only until every store but the newest + // (this one) has finished reading its sC buffer, so tile T's store drains + // during tile T+1's compute instead of stalling here. The buffer freed is + // the one from two tiles ago — safe to reuse next time cbuf lands on it. + __syncthreads(); // sC[cbuf] fully packed by the consumer + fence_async_shared(); // make the atomicXor writes visible to the async proxy + if (t == 0) { + tma_store_2d(&tma_c, col0 * 2, row0, sCb); // x in UINT32 units (2 per limb) + tma_store_commit(); + tma_store_wait_read1(); + } + __syncthreads(); // order the deferred wait before the next tile + // reuses (zeroes) an sC buffer + } + // Drain the last outstanding output store before the CTA exits. + if (t == 0) tma_store_wait(); +} + +// ── Device-resident packing kernels (BLAS3 GPU row-reduction port) ─────────── +// +// These reproduce, on device, the host operand pre-arrangement in src/lib.rs +// (pad_2d + interleave_a for A; pad_2d + transpose_b for B) so a GEMM can run +// over persistent device buffers with no host round-trip. One thread per output +// u64. Packing is lower-order work, so these favor clarity over peak bandwidth. +// They reference the tiling constants (TM, KL, NG, TK) above — the single source +// of truth shared with the compute kernel. + +typedef unsigned long long u64_t; + +// A: gather the natural row-major (m_orig × sa_orig) limb array into row-major +// K-major tiles (TM rows × KL u64), ordered K-chunk-major then M-tile-major. +// Rows/limbs past the real extent (padding M→m_padded, K→k_padded) read as zero. +// `sa_orig` is the logical K-limb count (columns to pack); `a_stride` is the +// source row stride in limbs, which may exceed sa_orig when A is a sub-block of +// a wider buffer (e.g. a wide-panel multiplier matrix L stored with stride bl +// but used with only ceil(pr/64) occupied limbs). +extern "C" __global__ void pack_a( + u64_t* __restrict__ out, const u64_t* __restrict__ a, + unsigned m_orig, unsigned sa_orig, unsigned a_stride, unsigned m_tiles, unsigned total) +{ + unsigned idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= total) return; + const unsigned tile_u64s = TM * KL; + unsigned tile_idx = idx / tile_u64s; + unsigned within = idx % tile_u64s; + unsigned row = within / KL; + unsigned kl = within % KL; + unsigned bi = tile_idx % m_tiles; + unsigned kk = tile_idx / m_tiles; + unsigned global_row = bi * TM + row; + unsigned global_kl = kk * KL + kl; + u64_t val = 0; + if (global_row < m_orig && global_kl < sa_orig) + val = a[(u64_t)global_row * a_stride + global_kl]; + out[idx] = val; +} + +// B: pad_2d(K→k_padded) + transpose_b. Natural (k_orig × n_lim) limb array → +// K-major bit-transposed tiles (NG*64 operand rows × KL u64), ordered K-chunk- +// major then column-group-major. Column groups whose limb ≥ n_lim stay zero. +extern "C" __global__ void pack_b( + u64_t* __restrict__ out, const u64_t* __restrict__ b, + unsigned k_orig, unsigned n_lim, unsigned n_groups, unsigned total) +{ + unsigned idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= total) return; + const unsigned tile = NG * 64 * KL; + unsigned tile_idx = idx / tile; + unsigned within = idx % tile; + unsigned j = within / KL; // operand row within the NG*64-row tile + unsigned kl = within % KL; + unsigned cg = tile_idx % n_groups; + unsigned kk = tile_idx / n_groups; + unsigned lg = j / 64; + unsigned jj = j % 64; + unsigned limb = cg * NG + lg; + if (limb >= n_lim) { out[idx] = 0; return; } + u64_t val = 0; + #pragma unroll + for (unsigned bit = 0; bit < 64; ++bit) { + unsigned br = kk * TK + kl * 64 + bit; + u64_t word = (br < k_orig) ? b[(u64_t)br * n_lim + limb] : 0; + val |= ((word >> jj) & 1ULL) << bit; + } + out[idx] = val; +} + +// Fused XOR-accumulate the padded GEMM output C (m × c_stride limbs/row) into a +// destination region of a persistent matrix (dst_stride limbs/row) starting at +// limb dst_limb: dst[j][dst_limb + col] ^= C[j][col], for j= m * width) return; + unsigned j = idx / width; + unsigned col = idx % width; + dst[(u64_t)j * dst_stride + dst_limb + col] ^= c[(u64_t)j * c_stride + col]; +} + +// ── Panel factorization kernel (BLAS3 GPU row-reduction port, design §5) ────── +// +// Forward panel factorization of ONE 64-bit column panel (limb `plimb`), the +// only column-indexed region of the reduction. A single CTA sweeps the 64 bit +// positions in order, with a __syncthreads between bits; the panel limb (m u64, +// a few MB) streams through L2. This is the b=64 base kernel of design §5(1): +// per bit, a find-first reduction picks the pivot (the lone column op), then a +// row-parallel masked XOR clears it from the rows *below* and records the +// multiplier bit into L. Forward-only (rows above pivots are left for the +// back-substitution pass), matching the CPU Step A in src/matrix/blas3.rs. +// +// Rows are addressed through the virtual permutation `perm` (design §4.3): a +// "row swap" swaps two perm entries; the matrix bytes never move. L is indexed +// by ORIGINAL row id (perm[p]), so it needs no swapping. Emitted to host: `pr` +// (pivots found) and `pivcols` (their absolute columns). L, the reduced panel, +// and perm stay on device. +// +// Launch with ONE block. THREADS threads (a power of two ≤ 1024). +extern "C" __global__ void panel_factor( + u64_t* __restrict__ m_buf, // m × stride limbs, in place + unsigned* __restrict__ perm, // length m, virtual row order + u64_t* __restrict__ l_buf, // m × l_stride limbs (multipliers), in place + unsigned* __restrict__ pivcols,// out: absolute pivot columns, length ≤ 64 + unsigned* __restrict__ pr_out, // out: pivots found in this panel (1 int) + unsigned plimb, unsigned r, unsigned n, + unsigned m, unsigned stride, unsigned l_stride) +{ + extern __shared__ int s_red[]; // blockDim ints for the min-reduction + __shared__ int s_pivpos; + __shared__ unsigned s_pr; + const int tid = threadIdx.x; + const int nt = blockDim.x; + if (tid == 0) s_pr = 0; + __syncthreads(); + + for (unsigned j = 0; j < 64; ++j) { + unsigned q = plimb * 64 + j; + if (q >= n) break; + unsigned pr = s_pr; + + // find-first: smallest position p in [r+pr, m) whose row has bit j set. + int local_min = 0x7fffffff; + for (unsigned p = r + pr + tid; p < m; p += nt) { + unsigned row = perm[p]; + if ((m_buf[(u64_t)row * stride + plimb] >> j) & 1ULL) + local_min = min(local_min, (int)p); + } + s_red[tid] = local_min; + __syncthreads(); + for (int off = nt / 2; off > 0; off >>= 1) { + if (tid < off) s_red[tid] = min(s_red[tid], s_red[tid + off]); + __syncthreads(); + } + if (tid == 0) s_pivpos = s_red[0]; + __syncthreads(); + if (s_pivpos == 0x7fffffff) continue; // free column: no pivot + + // Promote: swap the pivot row up to position r+pr (perm swap only). + if (tid == 0) { + unsigned a = r + pr, b = (unsigned)s_pivpos; + unsigned t = perm[a]; perm[a] = perm[b]; perm[b] = t; + pivcols[pr] = q; + s_pr = pr + 1; + } + __syncthreads(); + + unsigned pivrow = perm[r + pr]; + u64_t pivword = m_buf[(u64_t)pivrow * stride + plimb]; + + // Row-parallel masked XOR: clear bit j from the rows *below* the pivot, + // recording the multiplier bit pr into L[row]. + for (unsigned p = r + pr + 1 + tid; p < m; p += nt) { + unsigned row = perm[p]; + u64_t* cell = &m_buf[(u64_t)row * stride + plimb]; + if ((*cell >> j) & 1ULL) { + l_buf[(u64_t)row * l_stride + (pr >> 6)] |= (1ULL << (pr & 63)); + *cell ^= pivword; + } + } + __syncthreads(); + } + if (tid == 0) *pr_out = s_pr; +} + +// ── Multi-CTA (cooperative) panel factorization ────────────────────────────── +// +// The single-CTA `panel_factor` above uses one SM of ~132 and is the dominant +// cost of the forward pass (profiled ~76% of GPU time at n=2^15 half-rank). This +// version does the identical math but spreads each bit-step's find-first and +// masked-XOR across the *whole grid*: the launch is cooperative (all CTAs +// co-resident), so we can barrier the grid between the 64 sequential bit-steps. +// +// Grid barrier is a self-contained sense-counting spin (no cooperative_groups / +// cudadevrt dependency, so it compiles under `nvcc -ptx`): each CTA's leader +// thread __threadfence()s its global writes, atomically arrives at a shared +// counter, and spins until all `total_ctas` CTAs of the current round have +// arrived. `goal` (= round · total_ctas) is tracked in a register that every +// thread advances identically — control flow is grid-uniform (all CTAs branch +// on the same broadcast `g_min`), so the arrival counts always match. Requires +// co-residency, which the cooperative launch guarantees; `barrier` must be 0 at +// launch. +__device__ __forceinline__ void grid_sync(unsigned* barrier, unsigned goal) { + __syncthreads(); + __threadfence(); + if (threadIdx.x == 0) { + atomicAdd(barrier, 1u); // arrive (one RMW per CTA) + // Spin on a *plain* volatile load, not atomicAdd(...,0): a read-modify- + // write acquires the cache line exclusively every iteration, serializing + // ~1000 spinning CTAs on one line and making each barrier cost ~µs. A + // volatile load leaves the line shared, so the spin is nearly free. + while (*((volatile unsigned*)barrier) < goal) { /* wait for the grid */ } + } + __syncthreads(); +} + +// Wide-panel factorization (design §5(2) / CPU blas3.rs Step A): factor the b=bl·64 +// column panel at limbs [ppanel, ppanel+bl) in place, forward-only from pivot row +// r, grid-parallel across all bl·64 columns. Unlike a single-limb panel, the +// masked XOR clears each pivot from the below rows across **all bl panel limbs** +// (the intra-panel Schur update, done inline), so a later sub-column's find-first +// sees fully reduced bits and one wide trailing GEMM (K = pr ≤ b) fixes up the +// columns beyond the panel. Multipliers go into a bl-limb-wide L (indexed by pr). +// +// `scratch` is 3 u32: [0]=barrier (must be 0), [1]=g_min (int), [2]=g_pr. +// `g_pivword` is bl u64 (the pivot row's panel limbs, broadcast). Launch +// cooperatively with `total_ctas` = gridDim.x. bl==1 reproduces the single-limb +// kernel exactly. +extern "C" __global__ void panel_factor_coop( + u64_t* __restrict__ m_buf, + unsigned* __restrict__ perm, + u64_t* __restrict__ l_buf, + unsigned* __restrict__ pivcols, + unsigned* __restrict__ pr_out, + unsigned* __restrict__ scratch, // [barrier, g_min(int), g_pr] + u64_t* __restrict__ g_pivword, // broadcast pivot panel limbs (bl u64) + unsigned ppanel, unsigned bl, unsigned r, unsigned n, + unsigned m, unsigned stride, unsigned l_stride, + unsigned total_ctas) +{ + extern __shared__ int s_red[]; // blockDim ints for the CTA-local min-reduction + const int tid = threadIdx.x; + const int nt = blockDim.x; + const unsigned gtid = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned gnt = gridDim.x * blockDim.x; + + unsigned* barrier = &scratch[0]; + int* g_min = (int*)&scratch[1]; + unsigned* g_pr = &scratch[2]; + + if (gtid == 0) { *g_min = 0x7fffffff; *g_pr = 0; } + unsigned goal = 0; + goal += total_ctas; grid_sync(barrier, goal); // init visible grid-wide + + for (unsigned cc = 0; cc < bl * 64; ++cc) { + unsigned q = ppanel * 64 + cc; + if (q >= n) break; + unsigned plimb = ppanel + cc / 64; // absolute limb of column q + unsigned j = cc & 63; // bit within that limb + unsigned pr = *g_pr; + + // find-first: smallest position p in [r+pr, m) whose row has bit q set. + int local_min = 0x7fffffff; + for (unsigned p = r + pr + gtid; p < m; p += gnt) { + unsigned row = perm[p]; + if ((m_buf[(u64_t)row * stride + plimb] >> j) & 1ULL) + local_min = min(local_min, (int)p); + } + s_red[tid] = local_min; + __syncthreads(); + for (int off = nt / 2; off > 0; off >>= 1) { + if (tid < off) s_red[tid] = min(s_red[tid], s_red[tid + off]); + __syncthreads(); + } + if (tid == 0) atomicMin(g_min, s_red[0]); + goal += total_ctas; grid_sync(barrier, goal); // [A] all atomicMin done + + int pivpos = *g_min; + if (pivpos != 0x7fffffff) { + // Thread 0 reads the pivot row's bl panel limbs (broadcast via + // g_pivword), swaps the pivot up to position r+pr, resets g_min and + // advances g_pr. The displaced row lands at pivpos and is handled by + // the XOR loop (which reads a now-stable perm after [B]). + if (gtid == 0) { + unsigned pivrow = perm[pivpos]; + for (unsigned t = 0; t < bl; ++t) + g_pivword[t] = m_buf[(u64_t)pivrow * stride + ppanel + t]; + unsigned a = r + pr; + perm[pivpos] = perm[a]; perm[a] = pivrow; + pivcols[pr] = q; + *g_min = 0x7fffffff; // reset for next column + *g_pr = pr + 1; + } + goal += total_ctas; grid_sync(barrier, goal); // [B] swap + pivword + resets visible + + // masked XOR of the rows *below* the pivot, across ALL bl panel limbs. + for (unsigned p = r + pr + 1 + gtid; p < m; p += gnt) { + unsigned row = perm[p]; + u64_t* base = &m_buf[(u64_t)row * stride + ppanel]; + if ((base[cc / 64] >> j) & 1ULL) { + l_buf[(u64_t)row * l_stride + (pr >> 6)] |= (1ULL << (pr & 63)); + for (unsigned t = 0; t < bl; ++t) + base[t] ^= g_pivword[t]; + } + } + goal += total_ctas; grid_sync(barrier, goal); // [C] XOR done before next find + } + // free column (pivpos == INT_MAX): grid-uniform, no extra barriers. + } + if (gtid == 0) *pr_out = *g_pr; +} + +// ── Active-row compaction (design §8.2) ────────────────────────────────────── +// +// A below row that is entirely zero across the remaining columns [start_limb, +// stride) can never become a pivot and never contributes to a trailing update +// (its multiplier bits are always zero), so it is permanently dead. At half rank +// ~half the rows die during elimination. Marking them lets the driver shrink the +// row range panel_factor scans. `live[idx]` (idx over rows [r, m)) = 1 iff row +// perm[r+idx] has any set bit in limbs [start_limb, stride). +extern "C" __global__ void mark_live( + unsigned* __restrict__ live, const u64_t* __restrict__ m_buf, + const unsigned* __restrict__ perm, + unsigned r, unsigned m, unsigned start_limb, unsigned stride) +{ + unsigned idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= m - r) return; + unsigned row = perm[r + idx]; + u64_t acc = 0; + for (unsigned c = start_limb; c < stride; ++c) + acc |= m_buf[(u64_t)row * stride + c]; + live[idx] = (acc != 0) ? 1u : 0u; +} + +// ── Forward-pass driver kernels (BLAS3 GPU row-reduction port, design §4.4) ─── +// +// After panel_factor establishes `pr` pivots at perm positions [r, r+pr), the +// driver (1) promotes the pivot rows' trailing, (2) drops them from the +// multiplier matrix, (3) gathers them into a contiguous U for the trailing GEMM. + +// (1) Promote pivot-row trailings: realize the deferred trailing of each pivot +// by replaying the earlier this-panel pivots recorded in L. Sequential in k +// (pivot k uses the already-promoted pivots i> 6)] >> (i & 63)) & 1ULL) + acc ^= m_buf[(u64_t)perm[r + i] * stride + first_limb + c]; + } + m_buf[(u64_t)row_k * stride + first_limb + c] = acc; + } + } +} + +// Cooperative multi-CTA promotion (right-looking): identical result to +// promote_pivots but grid-parallel. Processing pivots i = 0..pr in order, once +// pivot i's trailing is final we add it to every later pivot k>i that carries its +// multiplier (L[perm[r+k]][i]) — M[k] ^= M[i] across all trailing limbs — so by +// the time we reach pivot i it is already fully promoted. Sequential in i (grid +// barrier between steps), but each step's XOR is flattened over (later-pivot × +// limb) across the whole grid, replacing the single-CTA kernel's ~pr² per-column +// serial replay. `barrier` must be 0 at launch; `cond` holds ≥ pr unsigned. +// `l_limb_off` offsets the L read so this can promote a sub-block [lo, lo+pr): +// the caller passes r = r0+lo and l_limb_off = lo/64 (lo is 64-aligned), so bit i +// here maps to global multiplier bit lo+i. Zero for a full-panel promote. +extern "C" __global__ void promote_coop( + u64_t* __restrict__ m_buf, const unsigned* __restrict__ perm, + const u64_t* __restrict__ l_buf, + unsigned r, unsigned pr, unsigned first_limb, unsigned trailing_limbs, + unsigned stride, unsigned l_stride, unsigned l_limb_off, + unsigned* __restrict__ barrier, unsigned* __restrict__ cond, + unsigned total_ctas) +{ + const unsigned gtid = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned gnt = gridDim.x * blockDim.x; + (void)cond; // multiplier bits are read straight from L (never mutated here) + unsigned goal = 0; + for (unsigned i = 0; i < pr; ++i) { + unsigned rowi = perm[r + i]; + unsigned nk = pr - i - 1; // later pivots k in (i, pr) + // The clear-condition reads L, which the XOR never writes, so we test it + // inline instead of gathering it under a separate barrier — one grid + // barrier per pivot instead of two (these kernels are barrier-bound). + unsigned total = nk * trailing_limbs; // fits u32 + for (unsigned idx = gtid; idx < total; idx += gnt) { + unsigned krel = idx / trailing_limbs; + unsigned rowk = perm[r + i + 1 + krel]; + if (!((l_buf[(u64_t)rowk * l_stride + l_limb_off + (i >> 6)] >> (i & 63)) & 1ULL)) + continue; + unsigned c = idx - krel * trailing_limbs; + m_buf[(u64_t)rowk * stride + first_limb + c] ^= + m_buf[(u64_t)rowi * stride + first_limb + c]; + } + goal += total_ctas; grid_sync(barrier, goal); // finish i before next reads M[i+1] + } +} + +// (2) Zero the L rows of the pr pivot rows so the trailing GEMM (which runs over +// all m rows) leaves them untouched — their trailing is already promoted. +extern "C" __global__ void zero_pivot_l( + const unsigned* __restrict__ perm, u64_t* __restrict__ l_buf, + unsigned r, unsigned pr, unsigned l_stride) +{ + unsigned idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= pr) return; + unsigned row = perm[r + idx]; + for (unsigned c = 0; c < l_stride; ++c) + l_buf[(u64_t)row * l_stride + c] = 0; +} + +// (3) Gather the pr pivot rows' trailing limbs [first_limb, first_limb+ncols) +// (through perm) into a contiguous pr × ncols buffer — the GEMM operand U. +extern "C" __global__ void gather_rows( + u64_t* __restrict__ dst, const u64_t* __restrict__ m_buf, + const unsigned* __restrict__ perm, + unsigned r, unsigned first_limb, unsigned pr, unsigned ncols, unsigned stride) +{ + unsigned idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= pr * ncols) return; + unsigned k = idx / ncols, c = idx % ncols; + dst[idx] = m_buf[(u64_t)perm[r + k] * stride + first_limb + c]; +} + +// ── Back-substitution kernels (BLAS3 GPU row-reduction port, design §4.6) ───── +// +// Echelon → RREF, blocked right-to-left over pivot blocks. For a block of pivots +// at perm positions [s, e): (1) reduce the block among itself, then (2) clear +// the block's pivot columns from all rows above [0, s) via one X·U GEMM. + +// (1) Reduce the pivot block [s, e) to RREF among itself: process pivots +// high-to-low, clearing pivot column pivcols[k] from the earlier block rows +// [s, k). One CTA; threads parallelize over limbs. Sequential in k (a row used +// as a source must already be fully reduced) — a __syncthreads separates the k +// steps, and one inside the j-loop orders every row's condition-read before any +// write to that row (the pivot bit itself is cleared by the XOR). +extern "C" __global__ void block_reduce_rref( + u64_t* __restrict__ m_buf, const unsigned* __restrict__ perm, + const unsigned* __restrict__ pivcols, + unsigned s, unsigned e, unsigned stride) +{ + // The block has ≤64 pivot rows. For pivot k (processed high-to-low) we clear + // its pivot column from every earlier block row j∈[s,k) that has the bit set, + // XORing rowk into rowj across the full row width. The condition for every + // such j is read at once into shared memory *before* any XOR (rowj[qlimb] is + // itself cleared by the XOR), then the (j, limb) work is flattened across all + // threads — two __syncthreads per pivot k instead of the previous ~bp² (one + // per (k,j) pair). Still one CTA; parallel over (j × limb). + __shared__ unsigned char cond[64]; // block size ≤ 64 + const int tid = threadIdx.x; + const int nt = blockDim.x; + for (unsigned k = e; k-- > s;) { + unsigned qk = pivcols[k]; + unsigned qlimb = qk >> 6, qbit = qk & 63; + unsigned rowk = perm[k]; + unsigned nj = k - s; // earlier block rows [s, k) + + // Gather the pivot-k bit of every earlier block row (pre-XOR). + for (unsigned j = tid; j < nj; j += nt) + cond[j] = (unsigned char)((m_buf[(u64_t)perm[s + j] * stride + qlimb] >> qbit) & 1ULL); + __syncthreads(); + + // Limb-parallel: each thread owns a set of columns c, loads rowk[c] once + // and XORs it into every flagged rowj at that column. No 64-bit division; + // rowk[c] reused across all ≤64 rows. Distinct c per thread ⇒ no races. + for (unsigned c = tid; c < stride; c += nt) { + u64_t rowk_c = m_buf[(u64_t)rowk * stride + c]; + for (unsigned j = 0; j < nj; ++j) + if (cond[j]) + m_buf[(u64_t)perm[s + j] * stride + c] ^= rowk_c; + } + __syncthreads(); // finish this k before the next reads the bits again + } +} + +// Cooperative multi-CTA version of block_reduce_rref: identical math, but the +// per-pivot clear is spread across the whole grid instead of one SM (the +// single-CTA kernel was ~22% of GPU time at n=2^16). Same structure — gather the +// clear-conditions for pivot k, barrier, XOR rowk into the flagged block rows +// across all limbs, barrier — but with a global `cond` buffer and the atomic grid +// barrier (grid-uniform k-loop, so arrival counts always match). The (j, limb) +// work is flattened over the grid for full-machine parallelism. `barrier` must be +// 0 at launch; `cond` holds ≥ (e-s) unsigned. Launch cooperatively. +extern "C" __global__ void block_reduce_coop( + u64_t* __restrict__ m_buf, const unsigned* __restrict__ perm, + const unsigned* __restrict__ pivcols, + unsigned s, unsigned e, unsigned stride, + unsigned* __restrict__ barrier, unsigned* __restrict__ cond, + unsigned total_ctas) +{ + const unsigned gtid = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned gnt = gridDim.x * blockDim.x; + unsigned goal = 0; + for (unsigned k = e; k-- > s;) { + unsigned qk = pivcols[k]; + unsigned qlimb = qk >> 6, qbit = qk & 63; + unsigned rowk = perm[k]; + unsigned nj = k - s; // earlier block rows [s, k) + + for (unsigned j = gtid; j < nj; j += gnt) + cond[j] = (unsigned)((m_buf[(u64_t)perm[s + j] * stride + qlimb] >> qbit) & 1ULL); + goal += total_ctas; grid_sync(barrier, goal); // [A] conds visible pre-XOR + + // XOR rowk into each flagged rowj, flattened over (j, limb) across the grid. + unsigned total = nj * stride; // ≤ 64 · stride, fits u32 + for (unsigned idx = gtid; idx < total; idx += gnt) { + unsigned j = idx / stride; + if (!cond[j]) continue; + unsigned c = idx - j * stride; + m_buf[(u64_t)perm[s + j] * stride + c] ^= m_buf[(u64_t)rowk * stride + c]; + } + goal += total_ctas; grid_sync(barrier, goal); // [B] finish k before next reads + } +} + +// (2a) Gather X: for rows at perm positions [0, s), the bits at the `count` +// block pivot columns pivcols[col_start .. col_start+count). One thread per +// (row, dst-limb) builds a full limb, so no atomics. dst is s × dst_stride. +extern "C" __global__ void gather_cols( + u64_t* __restrict__ dst, const u64_t* __restrict__ m_buf, + const unsigned* __restrict__ perm, const unsigned* __restrict__ pivcols, + unsigned col_start, unsigned s, unsigned count, + unsigned stride, unsigned dst_stride) +{ + unsigned idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= s * dst_stride) return; + unsigned jpos = idx / dst_stride, dl = idx % dst_stride; + unsigned row = perm[jpos]; + u64_t val = 0; + for (unsigned bb = 0; bb < 64; ++bb) { + unsigned i = dl * 64 + bb; + if (i >= count) break; + unsigned q = pivcols[col_start + i]; + if ((m_buf[(u64_t)row * stride + (q >> 6)] >> (q & 63)) & 1ULL) + val |= (1ULL << bb); + } + dst[idx] = val; +} + +// (2b) Scatter-XOR the GEMM result C (s × c_stride) into rows at perm positions +// [0, s): M[perm[jpos]][first_limb + col] ^= C[jpos][col]. +extern "C" __global__ void xor_into_perm( + u64_t* __restrict__ m_buf, const u64_t* __restrict__ c, + const unsigned* __restrict__ perm, + unsigned s, unsigned width, unsigned stride, unsigned first_limb, + unsigned c_stride) +{ + unsigned idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= s * width) return; + unsigned jpos = idx / width, col = idx % width; + m_buf[(u64_t)perm[jpos] * stride + first_limb + col] ^= c[(u64_t)jpos * c_stride + col]; +} diff --git a/ext/crates/fp-cuda/examples/bench_giant.rs b/ext/crates/fp-cuda/examples/bench_giant.rs new file mode 100644 index 0000000000..89e32ecdb0 --- /dev/null +++ b/ext/crates/fp-cuda/examples/bench_giant.rs @@ -0,0 +1,80 @@ +use std::time::Instant; + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::matmul_b1; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let (major, minor) = gpu.compute_capability()?; + println!("GPU: sm_{major}{minor} (H100)"); + println!(); + + let mut rng = rand::rng(); + + // Several GB per matrix: for an NxN binary matrix, storage = N*N/8 bytes + // N=131072: 131072^2/8 = 2 GB per matrix + // N=65536: 65536^2/8 = 512 MB per matrix + // N=32768: 32768^2/8 = 128 MB per matrix + // + // For F2 matmul: 2*N^3 bit-operations (N^3 ANDs + N^3 XORs) + // Binary TOPS = 2*N^3 / time_seconds / 1e12 + + // N*N/8 bytes per matrix: + // N=32768, K=32768: 128 MB each + // N=65536, K=65536: 512 MB each (1 GB pair) + // N=131072, K=32768: 512 MB each + // N=131072, K=65536: 1 GB each (2 GB pair) + // N=131072, K=131072: 2 GB each (4 GB pair) + // N=262144, K=65536: 2 GB each (4 GB pair) + // N=262144, K=131072: 4 GB each (8 GB pair) + for &(m, k_actual, n) in &[ + (32768usize, 32768usize, 32768usize), // 128 MB each, warmup + (65536, 65536, 65536), // 512 MB each + (131072, 65536, 131072), // 1 GB each + (131072, 131072, 131072), // 2 GB each + (262144, 65536, 262144), // 2 GB + 2 GB + (262144, 131072, 262144), // 4 GB each + ] { + let stride_a = k_actual.div_ceil(64); + let stride_c = n.div_ceil(64); + let a_elems = m * stride_a; + let b_elems = k_actual * stride_c; + let a_bytes = a_elems * 8; + let b_bytes = b_elems * 8; + + println!("=== {m} x {k_actual} x {n} ==="); + println!(" A: {m}x{k_actual} = {:.1} MB", a_bytes as f64 / 1e6); + println!(" B: {k_actual}x{n} = {:.1} MB", b_bytes as f64 / 1e6); + + let a_data: Vec = (0..a_elems).map(|_| rng.random()).collect(); + let b_data: Vec = (0..b_elems).map(|_| rng.random()).collect(); + let a = Matrix::from_data(TWO, m, k_actual, a_data); + let b = Matrix::from_data(TWO, k_actual, n, b_data); + + // Warmup + let _ = matmul_b1(&gpu, &a, &b)?; + + // Timed runs + let trials = 3; + let mut best_secs = f64::MAX; + for _ in 0..trials { + let t0 = Instant::now(); + let _ = matmul_b1(&gpu, &a, &b)?; + let elapsed = t0.elapsed().as_secs_f64(); + best_secs = best_secs.min(elapsed); + } + + // 2*M*N*K bit-ops (AND + XOR per element) + let bit_ops = 2.0 * (m as f64) * (n as f64) * (k_actual as f64); + let tops = bit_ops / best_secs / 1e12; + println!(" Time: {:.3} ms", best_secs * 1e3); + println!(" Binary TOPS: {:.2}", tops); + println!(); + } + + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/bench_kernel.rs b/ext/crates/fp-cuda/examples/bench_kernel.rs new file mode 100644 index 0000000000..173dc15222 --- /dev/null +++ b/ext/crates/fp-cuda/examples/bench_kernel.rs @@ -0,0 +1,86 @@ +use std::time::Instant; + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::matmul_b1; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let (major, minor) = gpu.compute_capability()?; + println!("GPU: sm_{major}{minor}"); + println!("Timing includes host serialization + H2D + kernel + D2H."); + println!(); + + let mut rng = rand::rng(); + + // Focus on compute-bound regime: large K maximizes wgmma fraction + for &(m, k, n) in &[ + (8192usize, 8192usize, 8192usize), + (16384, 16384, 16384), + (32768, 32768, 32768), + ] { + let stride_a = k.div_ceil(64); + let stride_b = n.div_ceil(64); + let a_bytes = m * stride_a * 8; + let b_bytes = k * stride_b * 8; + + println!("=== {m} x {k} x {n} ==="); + println!( + " A: {:.1} MB, B: {:.1} MB", + a_bytes as f64 / 1e6, + b_bytes as f64 / 1e6 + ); + + let a_data: Vec = (0..m * stride_a).map(|_| rng.random()).collect(); + let b_data: Vec = (0..k * stride_b).map(|_| rng.random()).collect(); + let a = Matrix::from_data(TWO, m, k, a_data); + let b = Matrix::from_data(TWO, k, n, b_data); + + // Warmup (includes compilation, allocation) + let _ = matmul_b1(&gpu, &a, &b)?; + + // Timed: end-to-end (host serial + H2D + kernel + D2H) + let trials = 3; + let mut best = f64::MAX; + for _ in 0..trials { + let t0 = Instant::now(); + let _ = matmul_b1(&gpu, &a, &b)?; + best = best.min(t0.elapsed().as_secs_f64()); + } + + let bit_ops = 2.0 * (m as f64) * (n as f64) * (k as f64); + let tops = bit_ops / best / 1e12; + + // Estimate host overhead: time just serialization + padding + let t_host = Instant::now(); + let _a_ser = { + let mut v = Vec::new(); + a.to_bytes(&mut v).unwrap(); + v + }; + let _b_ser = { + let mut v = Vec::new(); + b.to_bytes(&mut v).unwrap(); + v + }; + let host_ser_ms = t_host.elapsed().as_secs_f64() * 1e3; + + println!( + " End-to-end: {:.1} ms → {:.1} binary TOPS", + best * 1e3, + tops + ); + println!( + " Host serialization alone: {:.1} ms ({:.0}% of total)", + host_ser_ms, + host_ser_ms / (best * 1e3) * 100.0 + ); + println!(" K-chunks per CTA: {}", k.div_ceil(256)); + println!(); + } + + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/bench_kernel_only.rs b/ext/crates/fp-cuda/examples/bench_kernel_only.rs new file mode 100644 index 0000000000..133945ae87 --- /dev/null +++ b/ext/crates/fp-cuda/examples/bench_kernel_only.rs @@ -0,0 +1,69 @@ +//! Kernel-only throughput for `matmul_b1`. +//! +//! Unlike `bench_kernel` (end-to-end, host-serialization-bound) and the +//! `cargo bench` criterion harness (also end-to-end), this isolates the GPU +//! kernel: all host (de)serialization, the TMA-layout pre-arrangement, and the +//! H2D/D2H copies happen once, then only back-to-back kernel launches are +//! timed (see `matmul_b1_timed`). This is the apples-to-apples number to +//! compare against the ~100-binary-TOPS pre-swizzle kernel baseline. +//! +//! Run: `cargo run --release -p fp-cuda --example bench_kernel_only`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::{matmul_b1, matmul_b1_timed}; +use rand::Rng; + +fn binary_tops(m: usize, k: usize, n: usize, secs: f64) -> f64 { + 2.0 * (m as f64) * (n as f64) * (k as f64) / secs / 1e12 +} + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let (major, minor) = gpu.compute_capability()?; + println!("GPU: sm_{major}{minor}"); + println!("Kernel-only binary TOPS (host setup + H2D/D2H excluded):\n"); + + let mut rng = rand::rng(); + let mut make = |rows: usize, cols: usize| { + let data: Vec = (0..rows * cols.div_ceil(64)) + .map(|_| rng.random()) + .collect(); + Matrix::from_data(TWO, rows, cols, data) + }; + + for &(m, k, n, iters) in &[ + (4096usize, 4096, 4096, 50), + (8192, 8192, 8192, 30), + (16384, 16384, 16384, 10), + (32768, 32768, 32768, 5), + ] { + let a = make(m, k); + let b = make(k, n); + + // Bit-exact correctness check against the CPU path once per shape. + let cpu = &a * &b; + let (gpu_ref, _) = matmul_b1_timed(&gpu, &a, &b, 1)?; + let ok = cpu == gpu_ref; + // matmul_b1 (single launch) must agree with the timed multi-launch path. + let single = matmul_b1(&gpu, &a, &b)?; + let idempotent = single == gpu_ref; + + let (_, secs) = matmul_b1_timed(&gpu, &a, &b, iters)?; + println!( + " {m:>6} x {k:>6} x {n:>6}: {:>7.1} binary TOPS ({:>8.3} ms/launch, {iters} iters) \ + correct={ok} idempotent={idempotent}", + binary_tops(m, k, n, secs), + secs * 1e3, + ); + if !ok || !idempotent { + eprintln!(" CORRECTNESS FAILURE at {m}x{k}x{n}"); + std::process::exit(1); + } + } + + println!("\nH100 binary tensor-op peak is ~360,000 TOPS."); + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/bench_shapes.rs b/ext/crates/fp-cuda/examples/bench_shapes.rs new file mode 100644 index 0000000000..547546d674 --- /dev/null +++ b/ext/crates/fp-cuda/examples/bench_shapes.rs @@ -0,0 +1,92 @@ +//! Isolate *why* kernel-only throughput drops past N=16384: is it total size, or +//! the B operand spilling out of the 50 MB L2? +//! +//! Each B column-panel is reused across every M-tile, so the L2-reuse condition +//! is simply whether the whole B matrix (K*N/8 bytes) fits in L2. These shapes +//! hold FLOPs fixed while flipping "B fits in L2", which a pure size/occupancy +//! story cannot explain. +//! +//! Run: `cargo run --release -p fp-cuda --example bench_shapes`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::matmul_b1_timed; +use rand::Rng; + +fn binary_tops(m: usize, k: usize, n: usize, secs: f64) -> f64 { + 2.0 * (m as f64) * (n as f64) * (k as f64) / secs / 1e12 +} + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + // Assumed L2 capacity (H100 / H200 NVL ≈ 50 MB); not read from the device. + // Override when profiling a card with a different L2 size. + let l2_mb = 50.0; + println!( + "Assumed GPU L2 ~= {l2_mb} MB (H100/H200 NVL). B in L2 (bytes = K*N/8) governs \ + cross-M-tile reuse.\n" + ); + + let mut rng = rand::rng(); + let mut make = |rows: usize, cols: usize| { + let data: Vec = (0..rows * cols.div_ceil(64)) + .map(|_| rng.random()) + .collect(); + Matrix::from_data(TWO, rows, cols, data) + }; + + // (M, K, N, iters, note) + let shapes = [ + (16384usize, 16384, 16384, 10, "cube, B fits"), + (32768, 32768, 32768, 5, "cube, B spills"), + // Same FLOPs (1.76e13), only B-in-L2 differs: + ( + 65536, + 16384, + 16384, + 10, + "tall: huge M, B FITS (=16384^3 x4 FLOPs)", + ), + ( + 16384, + 16384, + 65536, + 5, + "wide: huge N, B SPILLS (same FLOPs as above)", + ), + ( + 16384, + 65536, + 16384, + 5, + "deep: huge K, B SPILLS (same FLOPs as above)", + ), + // B fits even at 2x the tall-case FLOPs: + ( + 131072, + 16384, + 16384, + 5, + "taller: M=128K, B FITS (=16384^3 x8 FLOPs)", + ), + ]; + + println!( + "{:>7} {:>7} {:>7} | {:>9} {:>6} | {:>9} | note", + "M", "K", "N", "B (MB)", "fits", "TOPS" + ); + for &(m, k, n, iters, note) in &shapes { + let b_mb = (k as f64) * (n as f64) / 8.0 / 1e6; + let a = make(m, k); + let b = make(k, n); + let (_, secs) = matmul_b1_timed(&gpu, &a, &b, iters)?; + println!( + "{m:>7} {k:>7} {n:>7} | {b_mb:>9.1} {:>6} | {:>9.1} | {note}", + if b_mb <= l2_mb { "yes" } else { "NO" }, + binary_tops(m, k, n, secs), + ); + } + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/common/mod.rs b/ext/crates/fp-cuda/examples/common/mod.rs new file mode 100644 index 0000000000..6bbc78e24d --- /dev/null +++ b/ext/crates/fp-cuda/examples/common/mod.rs @@ -0,0 +1,87 @@ +//! Shared `fp::Matrix` glue for the examples and benches. +//! +//! The `fp-cuda` library itself is fp-agnostic (it takes raw row-major limb +//! slices) so that the `fp` crate can depend on it without a dependency cycle. +//! These thin wrappers restore the `Matrix`-typed convenience the examples want, +//! using `fp` — which `fp-cuda` pulls in only as a dev-dependency. +//! +//! `#![allow(dead_code)]` because each example/bench uses a different subset. +#![allow(dead_code)] + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::{DeviceMatrix, GpuContext}; + +/// Row-major, K-major `u64` limbs, exactly as [`matmul_b1_raw`] expects: +/// `rows × columns.div_ceil(64)` limbs, one bit per entry, no inter-row padding. +/// +/// [`matmul_b1_raw`]: fp_cuda::matmul_b1_raw +pub fn to_limbs(m: &Matrix) -> Vec { + let stride = m.columns().div_ceil(64); + let mut bytes = Vec::with_capacity(m.rows() * stride * 8); + m.to_bytes(&mut bytes).expect("Vec writes never fail"); + let (chunks, _) = bytes.as_chunks::<8>(); + chunks.iter().map(|&c| u64::from_le_bytes(c)).collect() +} + +/// `Matrix`-typed wrapper over [`fp_cuda::matmul_b1_raw`]. +pub fn matmul_b1( + gpu: &GpuContext, + a: &Matrix, + b: &Matrix, +) -> Result> { + assert_eq!(a.prime(), TWO); + assert_eq!(b.prime(), TWO); + assert_eq!(a.columns(), b.rows()); + let (m, k, n) = (a.rows(), a.columns(), b.columns()); + let c = fp_cuda::matmul_b1_raw(gpu, &to_limbs(a), m, k, &to_limbs(b), n)?; + Ok(Matrix::from_data(TWO, m, n, c)) +} + +/// `Matrix`-typed wrapper over the device-resident path +/// [`fp_cuda::GpuContext::matmul_b1_dev_roundtrip`] (on-device packing + GEMM). +pub fn matmul_b1_dev( + gpu: &GpuContext, + a: &Matrix, + b: &Matrix, +) -> Result> { + assert_eq!(a.prime(), TWO); + assert_eq!(b.prime(), TWO); + assert_eq!(a.columns(), b.rows()); + let (m, k, n) = (a.rows(), a.columns(), b.columns()); + let c = gpu.matmul_b1_dev_roundtrip(&to_limbs(a), m, k, &to_limbs(b), n)?; + Ok(Matrix::from_data(TWO, m, n, c)) +} + +/// Upload an `fp::Matrix` to a device-resident [`DeviceMatrix`]. +pub fn upload_matrix( + gpu: &GpuContext, + m: &Matrix, +) -> Result> { + assert_eq!(m.prime(), TWO); + gpu.upload(&to_limbs(m), m.rows(), m.columns()) +} + +/// Download a [`DeviceMatrix`] back into an `fp::Matrix`. +pub fn download_matrix( + gpu: &GpuContext, + dm: &DeviceMatrix, +) -> Result> { + let limbs = gpu.download(dm)?; + Ok(Matrix::from_data(TWO, dm.rows, dm.cols, limbs)) +} + +/// `Matrix`-typed wrapper over [`fp_cuda::matmul_b1_raw_timed`]. +pub fn matmul_b1_timed( + gpu: &GpuContext, + a: &Matrix, + b: &Matrix, + time_iters: usize, +) -> Result<(Matrix, f64), Box> { + assert_eq!(a.prime(), TWO); + assert_eq!(b.prime(), TWO); + assert_eq!(a.columns(), b.rows()); + let (m, k, n) = (a.rows(), a.columns(), b.columns()); + let (c, secs) = + fp_cuda::matmul_b1_raw_timed(gpu, &to_limbs(a), m, k, &to_limbs(b), n, time_iters)?; + Ok((Matrix::from_data(TWO, m, n, c), secs)) +} diff --git a/ext/crates/fp-cuda/examples/forward_reduce_demo.rs b/ext/crates/fp-cuda/examples/forward_reduce_demo.rs new file mode 100644 index 0000000000..3aea539766 --- /dev/null +++ b/ext/crates/fp-cuda/examples/forward_reduce_demo.rs @@ -0,0 +1,111 @@ +//! Phase 5a gate (BLAS3-GPU-HANDOFF §5): the device forward pass +//! (`forward_reduce`) must produce a correct row-echelon form over the +//! persistent buffer. Validated against an independent oracle — `row_reduce`: +//! +//! 1. the forward pass applies only elementary row operations, so it preserves +//! row space: `row_reduce(device_M) == row_reduce(original)`; +//! 2. rank matches `row_reduce`'s rank, and the reported pivot columns match; +//! 3. the non-pivot rows (perm positions `[r, m)`) are zeroed. +//! +//! No hand-written mirror reference — the check is against the ground-truth CPU +//! reducer. (Phase 5b adds back-substitution and validates full RREF equality.) +//! +//! Run with `cargo run -p fp-cuda --example forward_reduce_demo`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::{download_matrix, upload_matrix}; +use rand::Rng; + +fn pivot_columns(m: &Matrix) -> Vec { + (0..m.columns()).filter(|&q| m.pivots()[q] >= 0).collect() +} + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + println!("=== fp-cuda forward_reduce (Phase 5a) demo ==="); + + let mut rng = rand::rng(); + let mut any_fail = false; + // (rows, cols, rank): rank=0 → full-rank random; rank>0 → rank-deficient. + let cases: &[(usize, usize, usize)] = &[ + (64, 64, 0), + (100, 130, 0), + (200, 200, 0), + (300, 130, 30), // rank-deficient + (500, 320, 50), // multiple panels, rank-deficient + (1000, 256, 0), + (1000, 500, 100), // rank ≈ m/5, several panels + (2000, 640, 200), + (64, 300, 0), // wide (many panels), full row rank + ]; + for &(rows, cols, rank) in cases { + let mm = if rank == 0 { + let v: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) + } else { + let a: Vec> = (0..rows) + .map(|_| (0..rank).map(|_| rng.random::() as u32).collect()) + .collect(); + let b: Vec> = (0..rank) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + &Matrix::from_vec(TWO, &a) * &Matrix::from_vec(TWO, &b) + }; + + // Reference: row_reduce of the original. + let mut reference = mm.clone(); + let ref_rank = reference.row_reduce(); + let ref_pivots = pivot_columns(&reference); + + // Device forward pass. + let mut dm = upload_matrix(&gpu, &mm)?; + let (perm, r, mut pivcols) = gpu.forward_reduce(&mut dm)?; + let dev_limbs = gpu.download(&dm)?; + let perm_host = gpu.download_u32(&perm)?; + let dev_m = download_matrix(&gpu, &dm)?; + + // (1) row-space preserved: row_reduce(device) == row_reduce(original). + let mut dev_rr = dev_m.clone(); + let dev_rr_rank = dev_rr.row_reduce(); + let ok_space = dev_rr == reference && dev_rr_rank == ref_rank; + // (2) rank + pivot columns. + pivcols.sort_unstable(); + let ok_rank = r == ref_rank; + let ok_piv = pivcols == ref_pivots; + // (3) non-pivot rows (perm[r..]) are zero. + let stride = cols.div_ceil(64); + let ok_zero = perm_host[r..].iter().all(|&row| { + let base = row as usize * stride; + dev_limbs[base..base + stride].iter().all(|&w| w == 0) + }); + + let ok = ok_space && ok_rank && ok_piv && ok_zero; + println!( + " rows={rows} cols={cols} rank={ref_rank}: {} (space {} rank {} piv {} zero {})", + if ok { "OK" } else { "MISMATCH" }, + yn(ok_space), + yn(ok_rank), + yn(ok_piv), + yn(ok_zero), + ); + if !ok { + any_fail = true; + } + } + + if any_fail { + println!("Some cases mismatched."); + std::process::exit(1); + } + println!("All cases matched (device forward pass == CPU row-echelon oracle)."); + Ok(()) +} + +fn yn(b: bool) -> &'static str { + if b { "ok" } else { "BAD" } +} diff --git a/ext/crates/fp-cuda/examples/gemm_xor_into_demo.rs b/ext/crates/fp-cuda/examples/gemm_xor_into_demo.rs new file mode 100644 index 0000000000..a59f71359b --- /dev/null +++ b/ext/crates/fp-cuda/examples/gemm_xor_into_demo.rs @@ -0,0 +1,86 @@ +//! Phase 3 gate (BLAS3-GPU-HANDOFF §5): the fused trailing-update epilogue +//! `gemm_xor_into` must reproduce the CPU blas3 Step B — `M[:, off:] ^= L·U`, +//! in place over a persistent device buffer — bit-for-bit. +//! +//! For each shape we build well-formed random `M` (m×N), `L` (m×k), `U` (k×t) +//! with `off + t == N` and `off` a limb boundary, compute the reference the way +//! `blas3.rs` does (whole-limb XOR of the CPU product `L·U` into M's trailing +//! limbs), run the device epilogue, and compare the downloaded buffer. +//! +//! Run with `cargo run -p fp-cuda --example gemm_xor_into_demo`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::{download_matrix, to_limbs, upload_matrix}; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + println!("=== fp-cuda gemm_xor_into (Step B) demo ==="); + + let mut rng = rand::rng(); + let mut make = |rows: usize, cols: usize| { + let v: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) + }; + + let mut any_fail = false; + // (m, k, off, t): N = off + t. `off` a multiple of 64; k = inner (pivots); + // t = trailing width. Includes off=0 (whole-matrix update), tiny k, large m. + let shapes: &[(usize, usize, usize, usize)] = &[ + (2048, 4, 0, 4), + (2048, 16, 64, 200), + (4096, 32, 128, 300), + (512, 100, 512, 700), + (4096, 3, 256, 257), + (100000, 8, 256, 500), + ]; + for &(m, k, off, t) in shapes { + let big_n = off + t; + let stride = big_n.div_ceil(64); + let gw = t.div_ceil(64); + let goff = off / 64; + + let mm = make(m, big_n); + let l = make(m, k); + let u = make(k, t); + + // CPU reference: whole-limb XOR of the product into M's trailing limbs. + let g = &l * &u; // m × t + let m_limbs = to_limbs(&mm); + let g_limbs = to_limbs(&g); + let mut reference = m_limbs.clone(); + for j in 0..m { + for c in 0..gw { + reference[j * stride + goff + c] ^= g_limbs[j * gw + c]; + } + } + + // Device epilogue over a persistent buffer. + let mut dm = upload_matrix(&gpu, &mm)?; + let dl = upload_matrix(&gpu, &l)?; + let du = upload_matrix(&gpu, &u)?; + gpu.gemm_xor_into(&mut dm, &dl, &du, off)?; + let out = to_limbs(&download_matrix(&gpu, &dm)?); + + let ok = out == reference; + println!( + " m={m} k={k} off={off} t={t} (N={big_n}): {}", + if ok { "OK" } else { "MISMATCH" } + ); + if !ok { + any_fail = true; + } + } + + if any_fail { + println!("Some shapes mismatched."); + std::process::exit(1); + } + println!("All shapes matched (device trailing-update == CPU Step B)."); + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/matmul_b1_demo.rs b/ext/crates/fp-cuda/examples/matmul_b1_demo.rs new file mode 100644 index 0000000000..ed59faa26c --- /dev/null +++ b/ext/crates/fp-cuda/examples/matmul_b1_demo.rs @@ -0,0 +1,61 @@ +//! Smoke test for the `fp-cuda` matmul kernel. +//! +//! Multiplies one pair of small F_2 matrices on the GPU and verifies the result against +//! `fp::blas`. Run with `cargo oxide run -p fp-cuda --example matmul_b1_demo`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::matmul_b1; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let (major, minor) = gpu.compute_capability()?; + println!("=== fp-cuda matmul_b1 demo ==="); + println!("GPU compute capability: sm_{major}{minor}"); + + let mut rng = rand::rng(); + let mut make = |rows: usize, cols: usize| { + let data: Vec = (0..rows * cols.div_ceil(64)) + .map(|_| rng.random()) + .collect(); + Matrix::from_data(TWO, rows, cols, data) + }; + + for &(m, k, n) in &[ + (64, 256, 64), + (128, 256, 128), + (256, 256, 256), + (512, 512, 512), + (1024, 1024, 1024), + (2048, 512, 2048), + (4096, 256, 4096), + (8192, 256, 8192), + ] { + let a = make(m, k); + let b = make(k, n); + let cpu = &a * &b; + let gpu_out = matmul_b1(&gpu, &a, &b)?; + let ok = cpu == gpu_out; + println!( + " {m}x{k} * {k}x{n}: {}", + if ok { "OK" } else { "MISMATCH" } + ); + if !ok { + let mut cb = Vec::new(); + cpu.to_bytes(&mut cb).unwrap(); + let mut gb = Vec::new(); + gpu_out.to_bytes(&mut gb).unwrap(); + let cv = u64::from_le_bytes(cb[..8].try_into().unwrap()); + let gv = u64::from_le_bytes(gb[..8].try_into().unwrap()); + println!(" row 0: cpu={cv:016x} gpu={gv:016x}"); + println!(" GPU all zeros: {}", gb.iter().all(|&b| b == 0)); + std::process::exit(1); + } + } + + println!("All shapes matched."); + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/matmul_b1_dev_demo.rs b/ext/crates/fp-cuda/examples/matmul_b1_dev_demo.rs new file mode 100644 index 0000000000..6eece35e25 --- /dev/null +++ b/ext/crates/fp-cuda/examples/matmul_b1_dev_demo.rs @@ -0,0 +1,85 @@ +//! Validation for the **device-resident** GEMM path (`matmul_b1_dev`): on-device +//! packing (interleave A, bit-transpose B) + the wgmma.b1 kernel, no host +//! round-trip of the operands. Checks the product bit-for-bit against both the +//! CPU reference (`fp::blas`) and the host packing path (`matmul_b1_raw`), so the +//! new device packing kernels are pinned to the already-validated oracle. +//! +//! Run with `cargo run -p fp-cuda --example matmul_b1_dev_demo`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::{matmul_b1, matmul_b1_dev}; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let (major, minor) = gpu.compute_capability()?; + println!("=== fp-cuda matmul_b1_dev (device-resident) demo ==="); + println!("GPU compute capability: sm_{major}{minor}"); + + let mut rng = rand::rng(); + // Build via from_vec with genuine 0/1 entries so bits past the last real + // column/row are masked to zero — the invariant every real fp::Matrix holds. + // (from_data with raw random limbs would leave garbage in the unused high + // bits, which the CPU and GPU multiplies legitimately treat differently.) + let mut make = |rows: usize, cols: usize| { + let v: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) + }; + + let mut any_fail = false; + // A spread of shapes: partial-limb columns (n not a multiple of 64), K below + // and above the 1024 TILE_K boundary, M straddling the 384-row cluster pad, + // and the same large shapes the host demo exercises. + for &(m, k, n) in &[ + (1, 64, 1), + (7, 3, 5), + (64, 256, 64), + (100, 100, 100), + (128, 200, 130), + (200, 65, 300), + (256, 256, 256), + (383, 1000, 257), + (512, 512, 512), + (1000, 100, 1000), + (1024, 1024, 1024), + (2048, 512, 2048), + (4096, 256, 4096), + // Large-M, tiny-K/N: the shapes the last row-reduction panels produce + // (L is m×pr, U is pr×t with pr, t small but m large). These must be + // exact for the port even though the tiny-M shapes above are not. + (2048, 1, 1), + (2048, 5, 3), + (4096, 3, 4097), + (5000, 2, 7), + (100000, 4, 4), + ] { + let a = make(m, k); + let b = make(k, n); + let cpu = &a * &b; + let host_gpu = matmul_b1(&gpu, &a, &b)?; + let dev_gpu = matmul_b1_dev(&gpu, &a, &b)?; + + let ok_cpu = cpu == dev_gpu; + let ok_host = host_gpu == dev_gpu; + println!( + " {m}x{k} * {k}x{n}: vs cpu {} | vs host-gemm {}", + if ok_cpu { "OK" } else { "MISMATCH" }, + if ok_host { "OK" } else { "MISMATCH" }, + ); + if !ok_cpu || !ok_host { + any_fail = true; + } + } + + if any_fail { + println!("Some shapes mismatched."); + std::process::exit(1); + } + println!("All shapes matched (device-resident packing is bit-exact)."); + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/panel_factor_demo.rs b/ext/crates/fp-cuda/examples/panel_factor_demo.rs new file mode 100644 index 0000000000..7c684e9759 --- /dev/null +++ b/ext/crates/fp-cuda/examples/panel_factor_demo.rs @@ -0,0 +1,169 @@ +//! Phase 4 gate (BLAS3-GPU-HANDOFF §4): the device `panel_factor` base kernel +//! (one 64-bit panel, forward-only, virtual perm) must match a CPU reference of +//! the identical panel-limb operation bit-for-bit — the reduced panel limb, the +//! `perm` ordering, the multiplier matrix `L`, and `(pr, pivcols)`. +//! +//! The CPU reference below is a direct transliteration of the CUDA kernel, so +//! this pins the kernel to an executable spec. (Phase 5 wires it, promotion, and +//! gemm_xor_into into the full row reduction and validates against `row_reduce`.) +//! +//! Run with `cargo run -p fp-cuda --example panel_factor_demo`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::{download_matrix, to_limbs, upload_matrix}; +use rand::Rng; + +/// CPU mirror of the `panel_factor` CUDA kernel: forward factorization of the +/// single limb `plimb`, rows addressed through `perm`, starting at pivot row +/// `r`. Mutates `m_buf` (panel limb of below rows), `perm` (pivot swaps) and +/// `l_buf` (multiplier bits, by original row id). Returns `(pr, pivcols)`. +// The explicit `perm[p]` indexing deliberately mirrors the CUDA kernel. +#[allow(clippy::too_many_arguments, clippy::needless_range_loop)] +fn panel_factor_ref( + m_buf: &mut [u64], + perm: &mut [u32], + l_buf: &mut [u64], + plimb: usize, + r: usize, + n: usize, + m: usize, + stride: usize, + l_stride: usize, +) -> (usize, Vec) { + let mut pr = 0usize; + let mut pivcols = Vec::new(); + for j in 0..64 { + let q = plimb * 64 + j; + if q >= n { + break; + } + let mut pivpos = None; + for p in (r + pr)..m { + let row = perm[p] as usize; + if (m_buf[row * stride + plimb] >> j) & 1 == 1 { + pivpos = Some(p); + break; + } + } + let Some(pp) = pivpos else { continue }; + perm.swap(r + pr, pp); + pivcols.push(q as u32); + let pivrow = perm[r + pr] as usize; + let pivword = m_buf[pivrow * stride + plimb]; + for p in (r + pr + 1)..m { + let row = perm[p] as usize; + if (m_buf[row * stride + plimb] >> j) & 1 == 1 { + l_buf[row * l_stride + pr / 64] |= 1 << (pr % 64); + m_buf[row * stride + plimb] ^= pivword; + } + } + pr += 1; + } + (pr, pivcols) +} + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + println!("=== fp-cuda panel_factor (Phase 4) demo ==="); + + let mut rng = rand::rng(); + let mut any_fail = false; + // (rows, cols, plimb, r, rank): panel is limb `plimb`; factor rows [r, m). + // rank=0 → full-rank random; rank>0 → rank-deficient (free columns). + let cases: &[(usize, usize, usize, usize, usize)] = &[ + (64, 64, 0, 0, 0), + (200, 64, 0, 0, 0), + (1000, 130, 1, 0, 0), // second panel, partial last limb + (1000, 130, 1, 5, 0), // mid-sweep start row + (5000, 256, 2, 0, 0), + (5000, 256, 3, 40, 0), // last panel, r>0 + (100000, 64, 0, 0, 0), // large m + (300, 40, 0, 0, 0), // n < 64 (partial panel) + (500, 64, 0, 0, 10), // rank-deficient: ~10 pivots, free columns + (2000, 128, 1, 0, 20), // rank-deficient, second panel + (5000, 64, 0, 3, 5), // rank-deficient, mid-sweep start + ]; + for &(rows, cols, plimb, r, rank) in cases { + let stride = cols.div_ceil(64); + let mm = if rank == 0 { + make(&mut rng, rows, cols) + } else { + make_low_rank(&mut rng, rows, cols, rank) + }; + + // CPU reference. + let mut m_cpu = to_limbs(&mm); + let mut perm_cpu: Vec = (0..rows as u32).collect(); + let mut l_cpu = vec![0u64; rows]; // l_stride = 1 (≤64 pivots) + let (pr_cpu, piv_cpu) = panel_factor_ref( + &mut m_cpu, + &mut perm_cpu, + &mut l_cpu, + plimb, + r, + cols, + rows, + stride, + 1, + ); + + // Device. + let mut dm = upload_matrix(&gpu, &mm)?; + let mut perm = gpu.identity_perm(rows)?; + let mut dl = upload_matrix(&gpu, &Matrix::new(TWO, rows, 64))?; // zeroed, l_stride=1 + let (pr_gpu, piv_gpu) = gpu.panel_factor(&mut dm, &mut perm, &mut dl, plimb, r)?; + + let m_gpu = to_limbs(&download_matrix(&gpu, &dm)?); + let perm_gpu = gpu.download_u32(&perm)?; + let l_gpu = gpu.download(&dl)?; + + let ok_pr = pr_cpu == pr_gpu; + let ok_piv = piv_cpu == piv_gpu; + let ok_m = m_cpu == m_gpu; + let ok_perm = perm_cpu == perm_gpu; + let ok_l = l_cpu == l_gpu; + let ok = ok_pr && ok_piv && ok_m && ok_perm && ok_l; + println!( + " rows={rows} cols={cols} plimb={plimb} r={r} pr={pr_cpu}: {} (M {} perm {} L {} pr \ + {} piv {})", + if ok { "OK" } else { "MISMATCH" }, + yn(ok_m), + yn(ok_perm), + yn(ok_l), + yn(ok_pr), + yn(ok_piv), + ); + if !ok { + any_fail = true; + } + } + + if any_fail { + println!("Some cases mismatched."); + std::process::exit(1); + } + println!("All cases matched (device panel_factor == CPU reference)."); + Ok(()) +} + +fn yn(b: bool) -> &'static str { + if b { "ok" } else { "BAD" } +} + +fn make(rng: &mut impl Rng, rows: usize, cols: usize) -> Matrix { + let v: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) +} + +/// A rank-deficient matrix (rank ≤ `rank`) so the panel has free columns and the +/// kernel's find-first "no pivot" branch is exercised. +fn make_low_rank(rng: &mut impl Rng, rows: usize, cols: usize, rank: usize) -> Matrix { + let a = make(rng, rows, rank); + let b = make(rng, rank, cols); + &a * &b +} diff --git a/ext/crates/fp-cuda/examples/probe_k.rs b/ext/crates/fp-cuda/examples/probe_k.rs new file mode 100644 index 0000000000..c3893322fe --- /dev/null +++ b/ext/crates/fp-cuda/examples/probe_k.rs @@ -0,0 +1,38 @@ +//! Probe the GEMM's correctness as a function of the contraction dimension `k`, +//! at fixed large `m`, `n`. Isolates the small-`k` mismatch seen in +//! `matmul_b1_dev_demo`. Uses explicit 0/1 matrices (no high-bit garbage past +//! the last real column) so a mismatch can only be the kernel, not test data. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::matmul_b1; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let mut rng = rand::rng(); + // Build via from_vec with genuine 0/1 entries: Matrix masks unused bits. + let mut make = |rows: usize, cols: usize| { + let v: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) + }; + let (m, n) = (4096usize, 256usize); + println!("m={m} n={n}, sweeping k:"); + for k in [ + 1, 2, 4, 8, 16, 32, 48, 56, 60, 62, 63, 64, 65, 66, 96, 128, 256, + ] { + let a = make(m, k); + let b = make(k, n); + let cpu = &a * &b; + let gpu_out = matmul_b1(&gpu, &a, &b)?; + println!( + " k={k:4}: {}", + if cpu == gpu_out { "OK" } else { "MISMATCH" } + ); + } + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/probe_lowrank.rs b/ext/crates/fp-cuda/examples/probe_lowrank.rs new file mode 100644 index 0000000000..bc61cf9dac --- /dev/null +++ b/ext/crates/fp-cuda/examples/probe_lowrank.rs @@ -0,0 +1,42 @@ +//! Quick check of active-row compaction on a low-rank tall matrix: rank r ≪ m, +//! so once the first panels exhaust the rank every remaining below row is dead. +//! Compaction should drop the per-panel scan cost sharply. Compares device wall +//! time with compaction on vs off (set FP_CUDA_NO_COMPACT in the process to test +//! off — here we time the default-on path and print it; run twice to compare). + +use std::time::Instant; + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::upload_matrix; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let mut rng = rand::rng(); + // rank-r m×n matrix = (m×r random) · (r×n random). + let (m, n, r) = (65536usize, 16384usize, 1024usize); + let mut build = |rows: usize, cols: usize| -> Matrix { + let v: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) + }; + let a = build(m, r); + let b = build(r, n); + let mm = &a * &b; // rank ≤ r + + let t0 = Instant::now(); + let mut dm = upload_matrix(&gpu, &mm)?; + let (_perm, rank, _piv) = gpu.row_reduce_dev(&mut dm)?; + let secs = t0.elapsed().as_secs_f64(); + let compact = if std::env::var("FP_CUDA_NO_COMPACT").is_ok() { + "OFF" + } else { + "ON " + }; + println!("m={m} n={n} target_rank={r}: compaction {compact} device {secs:.3}s rank={rank}"); + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/probe_pr.rs b/ext/crates/fp-cuda/examples/probe_pr.rs new file mode 100644 index 0000000000..6791dc8eca --- /dev/null +++ b/ext/crates/fp-cuda/examples/probe_pr.rs @@ -0,0 +1,74 @@ +//! P0-a — flat-K microbench (Pass 0 of the optimization plan). +//! +//! The trailing GEMM of the row reduction is `C = L(m×pr) · U(pr×T)` where +//! `pr` = pivots-per-panel ≤ 64. But `matmul_b1_dev` pads the contraction +//! dimension to `TILE_K = 1024` (`k.next_multiple_of(TILE_K)`), so the tensor +//! cores issue a full K=1024 GEMM regardless of `pr`. This probe measures +//! **kernel-only** wall time as a function of `pr` at fixed `m, T`, using the +//! same `matmul_b1_raw_timed` path (host pack, but the timed loop is GPU GEMM +//! only). If wall time is flat from pr=64 to pr=1024, Loss 1 (16× K-padding) is +//! confirmed: ~15/16 of tensor-core issue is multiply-by-zero, and widening the +//! panel (raising `pr` toward `b`) is worth up to ~16× on the trailing GEMM. +//! +//! The effective-throughput column divides by the *real* work `2·m·T·pr`, so a +//! flat wall time shows Tbop/s climbing ~linearly with `pr` — exactly the +//! headroom Pass 3 reclaims. +//! +//! Run with `cargo run --release -p fp-cuda --example probe_pr`. + +use fp_cuda::{GpuContext, matmul_b1_raw_timed}; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let mut rng = rand::rng(); + + // Fixed trailing-update shape: tall-skinny GEMM like the real workload at + // n≈32768 (m rows still active, T trailing columns). + let m: usize = std::env::var("PROBE_M") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(32768); + let t: usize = std::env::var("PROBE_T") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(16384); + let iters: usize = 20; + let t_lim = t.div_ceil(64); + + println!("=== P0-a flat-K probe: C = L(m×pr)·U(pr×T), m={m} T={t}, {iters} iters ==="); + println!( + "{:>6} {:>12} {:>12} {:>10}", + "pr", "kernel(ms)", "Tbop/s", "vs pr=64" + ); + + // Small pr (≤1024) all pad to TILE_K=1024, so time is flat — the K-padding + // signature. Large pr (≥1024) uses nchunks = pr/1024 real K-tiles: if kernel + // time then scales ~linearly with pr, the kernel is K-work bound (each K-tile + // costs a fixed load+compute), which means a native small-TILE_K kernel would + // cut time proportionally — the payoff that justifies the fork. If large-pr + // time is instead sublinear/flat, a fixed per-launch overhead dominates and + // the fork would not help. + let pr_list = std::env::var("PROBE_LARGE") + .map(|_| vec![1024usize, 2048, 3072]) + .unwrap_or_else(|_| vec![64, 128, 256, 512, 1024]); + let mut base_ms = 0.0f64; + for (i, &pr) in pr_list.iter().enumerate() { + let a_lim = pr.div_ceil(64); + let a: Vec = (0..m * a_lim).map(|_| rng.random()).collect(); + let b: Vec = (0..pr * t_lim).map(|_| rng.random()).collect(); + let (_c, secs) = matmul_b1_raw_timed(&gpu, &a, m, pr, &b, t, iters)?; + let ms = secs * 1e3; + if i == 0 { + base_ms = ms; + } + let bops = 2.0 * m as f64 * t as f64 * pr as f64; + let tbops = bops / secs / 1e12; + println!("{pr:>6} {ms:>12.3} {tbops:>12.1} {:>9.2}x", ms / base_ms); + } + println!( + "\nFlat kernel(ms) across pr ⇒ Loss 1 confirmed (K padded to {}).", + 1024 + ); + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/prof_sizes.rs b/ext/crates/fp-cuda/examples/prof_sizes.rs new file mode 100644 index 0000000000..e2fb8846ed --- /dev/null +++ b/ext/crates/fp-cuda/examples/prof_sizes.rs @@ -0,0 +1,31 @@ +//! Minimal profiling target: exactly one `matmul_b1` kernel launch at each of +//! two sizes (16384, 32768), no CPU cross-check. Built for `ncu --launch-count` +//! so the L2-cliff hypothesis can be checked with hardware counters. +//! +//! `ncu --set basic --launch-count 2 target/release/examples/prof_sizes` + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::matmul_b1; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let mut rng = rand::rng(); + let mut make = |rows: usize, cols: usize| { + let data: Vec = (0..rows * cols.div_ceil(64)) + .map(|_| rng.random()) + .collect(); + Matrix::from_data(TWO, rows, cols, data) + }; + + for &n in &[16384usize, 32768] { + let a = make(n, n); + let b = make(n, n); + let _ = matmul_b1(&gpu, &a, &b)?; + eprintln!("launched {n}"); + } + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/reduce_pow2.rs b/ext/crates/fp-cuda/examples/reduce_pow2.rs new file mode 100644 index 0000000000..66310f12a8 --- /dev/null +++ b/ext/crates/fp-cuda/examples/reduce_pow2.rs @@ -0,0 +1,92 @@ +//! Scaling comparison of the device-resident reduction vs the two CPU reducers +//! (M4RI `row_reduce` and the CPU BLAS3 `row_reduce_blas3`) on random square +//! F₂ matrices of size 2^n, n = 10..=15. Random square matrices over F₂ are +//! essentially full rank, so this is the full-rank regime (work ∝ rank = n); +//! the target workload is half-rank, where the device lead is larger. +//! +//! The CPU baselines run multi-threaded (the example's `fp` enables +//! `concurrent`); the device path is reached directly, never through fp's GPU +//! dispatch. Correctness is asserted each size (device RREF == M4RI). +//! +//! Sizes are multiples of 64, so `from_data` with random limbs is well-formed. +//! +//! Run with `cargo run --release -p fp-cuda --example reduce_pow2`. + +use std::time::Instant; + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::upload_matrix; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let max_n: u32 = std::env::var("REDUCE_POW2_MAX") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(15); + println!("=== device vs CPU (M4RI, BLAS3) — random square 2^n, n=10..={max_n} ==="); + println!( + "{:>7} {:>10} {:>10} {:>10} {:>9} {:>9} {:>9}", + "n", "device(s)", "m4ri(s)", "blas3(s)", "vs-m4ri", "vs-blas3", "Tbop/s" + ); + + let mut rng = rand::rng(); + for e in 10..=max_n { + let n = 1usize << e; + let stride = n / 64; // n is a multiple of 64 + let data: Vec = (0..n * stride).map(|_| rng.random()).collect(); + let mm = Matrix::from_data(TWO, n, n, data); + + // Device: upload + full reduce + sync (excludes download). + let t0 = Instant::now(); + let mut dm = upload_matrix(&gpu, &mm)?; + let (perm, r, _piv) = gpu.row_reduce_dev(&mut dm)?; + let dev_secs = t0.elapsed().as_secs_f64(); + + // CPU M4RI (pure CPU, multi-threaded). + let mut m4ri = mm.clone(); + let t1 = Instant::now(); + let m4ri_rank = m4ri.row_reduce(); + let m4ri_secs = t1.elapsed().as_secs_f64(); + + // CPU BLAS3 (pure CPU, multi-threaded). + let mut blas3 = mm.clone(); + let t2 = Instant::now(); + let _ = blas3.row_reduce_blas3(); + let blas3_secs = t2.elapsed().as_secs_f64(); + + // Correctness: materialize device RREF (pivot k at row k via perm) and + // compare to the M4RI result. + let dev_limbs = gpu.download(&dm)?; + let perm_host = gpu.download_u32(&perm)?; + let mut e_limbs = vec![0u64; n * stride]; + for k in 0..r { + let src = perm_host[k] as usize * stride; + e_limbs[k * stride..k * stride + stride].copy_from_slice(&dev_limbs[src..src + stride]); + } + let dev_rref = Matrix::from_data(TWO, n, n, e_limbs); + let ok = r == m4ri_rank && dev_rref == m4ri; + + // Effective binary-op throughput: leading-order elimination work + // 2·m·n·R (bit-MAC = AND + accumulate = 2 ops). Tbop/s = 10^12 ops/s. + let tbops = 2.0 * n as f64 * n as f64 * r as f64 / dev_secs / 1e12; + println!( + "{:>7} {:>10.3} {:>10.3} {:>10.3} {:>8.2}x {:>8.2}x {:>9.1} rank={r} [{}]", + n, + dev_secs, + m4ri_secs, + blas3_secs, + m4ri_secs / dev_secs, + blas3_secs / dev_secs, + tbops, + if ok { "ok" } else { "WRONG" }, + ); + if !ok { + std::process::exit(1); + } + } + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/reduce_pow2_half.rs b/ext/crates/fp-cuda/examples/reduce_pow2_half.rs new file mode 100644 index 0000000000..0e5aa5dd3a --- /dev/null +++ b/ext/crates/fp-cuda/examples/reduce_pow2_half.rs @@ -0,0 +1,182 @@ +//! Half-rank scaling comparison (the target workload): device-resident +//! reduction vs the CPU reducers on rank-(n/2) square F₂ matrices of size 2^n. +//! Extends to 2^18; the CPU baselines are skipped once they would take too long +//! (M4RI is ~size-driven and explodes; CPU BLAS3 is ∝ rank and stays feasible +//! longer), leaving the device GPU-only at the top end. +//! +//! Policy (override via env EXP_MIN/EXP_MAX, M4RI_MAX_EXP, BLAS3_MAX_EXP): +//! device : always +//! M4RI : exp ≤ M4RI_MAX_EXP (default 16; ~48 min at 2^17, ~10 h at 2^18) +//! BLAS3 : exp ≤ BLAS3_MAX_EXP (default 17; ~1 h at 2^18) +//! +//! Correctness oracle per size: device RREF == M4RI (when run), else == CPU +//! BLAS3 (when run), else — GPU-only — the known rank n/2 plus idempotence +//! (re-reducing the RREF is a fixed point). Bit-exactness vs the CPU is already +//! validated exhaustively at smaller sizes (`row_reduce_demo`, `cuda_dispatch`). +//! +//! The half-rank matrix is built cheaply in packed form: n/2 random basis rows, +//! then n/2 rows each the XOR of an adjacent basis pair — rank n/2, ~50% dense, +//! O(n·stride) to construct. Sizes are multiples of 64 so limbs are well-formed. +//! +//! Run with `cargo run --release -p fp-cuda --example reduce_pow2_half`. + +use std::time::Instant; + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::upload_matrix; +use rand::Rng; + +fn env_exp(key: &str, default: u32) -> u32 { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Build a rank-(n/2) n×n F₂ matrix in packed form (n a multiple of 64). +fn half_rank(rng: &mut impl Rng, n: usize) -> Matrix { + let stride = n / 64; + let h = n / 2; + let mut data = vec![0u64; n * stride]; + // Top half: random basis rows. + for w in data[..h * stride].iter_mut() { + *w = rng.random(); + } + // Bottom half: row h+i = basis[i] XOR basis[(i+1) mod h]. + for i in 0..h { + let a = i * stride; + let b = ((i + 1) % h) * stride; + let dst = (h + i) * stride; + for c in 0..stride { + data[dst + c] = data[a + c] ^ data[b + c]; + } + } + Matrix::from_data(TWO, n, n, data) +} + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let exp_min = env_exp("EXP_MIN", 10); + let exp_max = env_exp("EXP_MAX", 18); + let m4ri_max = env_exp("M4RI_MAX_EXP", 16); + let blas3_max = env_exp("BLAS3_MAX_EXP", 17); + + println!("=== device vs CPU (M4RI, BLAS3) — half-rank square 2^n, n={exp_min}..={exp_max} ==="); + println!( + "{:>7} {:>10} {:>10} {:>10} {:>9} {:>9} {:>8} {:>9} check", + "n", "device(s)", "m4ri(s)", "blas3(s)", "vs-m4ri", "vs-blas3", "rank", "Tbop/s" + ); + + let mut rng = rand::rng(); + for exp in exp_min..=exp_max { + let n = 1usize << exp; + let stride = n / 64; + let mm = half_rank(&mut rng, n); + + // Device: upload + full reduce + sync. + let t0 = Instant::now(); + let mut dm = upload_matrix(&gpu, &mm)?; + let (perm, r, _piv) = gpu.row_reduce_dev(&mut dm)?; + let dev_secs = t0.elapsed().as_secs_f64(); + + // Materialize device RREF (pivot k = device row perm[k]). + let dev_limbs = gpu.download(&dm)?; + let perm_host = gpu.download_u32(&perm)?; + let mut e_limbs = vec![0u64; n * stride]; + for k in 0..r { + let src = perm_host[k] as usize * stride; + e_limbs[k * stride..k * stride + stride].copy_from_slice(&dev_limbs[src..src + stride]); + } + let dev_rref = Matrix::from_data(TWO, n, n, e_limbs); + + // CPU baselines (skipped past their thresholds). + let (m4ri_secs, m4ri_ok) = if exp <= m4ri_max { + let mut c = mm.clone(); + let t = Instant::now(); + let rank = c.row_reduce(); + ( + Some(t.elapsed().as_secs_f64()), + Some(rank == r && c == dev_rref), + ) + } else { + (None, None) + }; + let (blas3_secs, blas3_ok) = if exp <= blas3_max { + let mut c = mm.clone(); + let t = Instant::now(); + let rank = c.row_reduce_blas3(); + ( + Some(t.elapsed().as_secs_f64()), + Some(rank == r && c == dev_rref), + ) + } else { + (None, None) + }; + + // Correctness verdict: prefer a CPU oracle; else known rank + idempotence. + let check = if let Some(ok) = m4ri_ok { + if ok { + "vs-m4ri ok".to_string() + } else { + "vs-m4ri BAD".to_string() + } + } else if let Some(ok) = blas3_ok { + if ok { + "vs-blas3 ok".to_string() + } else { + "vs-blas3 BAD".to_string() + } + } else { + // GPU-only: rank must equal n/2 (known by construction), and the RREF + // must be a fixed point of a second device reduction. + let mut dm2 = upload_matrix(&gpu, &dev_rref)?; + let (perm2, r2, _) = gpu.row_reduce_dev(&mut dm2)?; + let l2 = gpu.download(&dm2)?; + let p2 = gpu.download_u32(&perm2)?; + let mut e2 = vec![0u64; n * stride]; + for k in 0..r2 { + let src = p2[k] as usize * stride; + e2[k * stride..k * stride + stride].copy_from_slice(&l2[src..src + stride]); + } + let idem = r2 == r && Matrix::from_data(TWO, n, n, e2) == dev_rref; + if r == n / 2 && idem { + "rank=n/2,idem ok".to_string() + } else { + format!("GPU-only BAD (r={r} idem={idem})") + } + }; + + let f = |o: Option| { + o.map(|s| format!("{s:.3}")) + .unwrap_or_else(|| " skip".into()) + }; + let ratio = |o: Option| { + o.map(|s| format!("{:.2}x", s / dev_secs)) + .unwrap_or_else(|| " —".into()) + }; + // Effective binary-op throughput: leading-order elimination work + // 2·m·n·R (each bit-MAC = AND + accumulate = 2 ops), the binary-TOPS + // convention. Tbop/s = 10^12 binary ops per second. + let bops = 2.0 * n as f64 * n as f64 * r as f64; + let tbops = bops / dev_secs / 1e12; + println!( + "{:>7} {:>10.3} {:>10} {:>10} {:>9} {:>9} {:>8} {:>9.1} {}", + n, + dev_secs, + f(m4ri_secs), + f(blas3_secs), + ratio(m4ri_secs), + ratio(blas3_secs), + r, + tbops, + check, + ); + if check.contains("BAD") { + std::process::exit(1); + } + } + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/reduce_timing.rs b/ext/crates/fp-cuda/examples/reduce_timing.rs new file mode 100644 index 0000000000..2db65a3afd --- /dev/null +++ b/ext/crates/fp-cuda/examples/reduce_timing.rs @@ -0,0 +1,69 @@ +//! Rough wall-clock comparison of the device-resident reduction against the CPU +//! BLAS3 reducer, on half-rank square matrices (the target regime shape). Not a +//! rigorous benchmark — a single timed run per size to see whether the device +//! path is in the right ballpark and where the time goes. Correctness is still +//! asserted (device RREF == CPU) so a fast-but-wrong result can't slip through. +//! +//! Run with `cargo run --release -p fp-cuda --example reduce_timing`. + +use std::time::Instant; + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::upload_matrix; +use rand::Rng; + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + println!("=== device vs CPU BLAS3 reduction (half-rank square) ==="); + + let mut rng = rand::rng(); + for &n in &[1024usize, 2048, 4096, 8192] { + let rank = n / 2; + // Half-rank n×n via A(n×rank)·B(rank×n). + let a: Vec> = (0..n) + .map(|_| (0..rank).map(|_| rng.random::() as u32).collect()) + .collect(); + let b: Vec> = (0..rank) + .map(|_| (0..n).map(|_| rng.random::() as u32).collect()) + .collect(); + let mm = &Matrix::from_vec(TWO, &a) * &Matrix::from_vec(TWO, &b); + + // Device: time upload + full reduce + sync (excludes download). + let t0 = Instant::now(); + let mut dm = upload_matrix(&gpu, &mm)?; + let (perm, r, _piv) = gpu.row_reduce_dev(&mut dm)?; + let dev_secs = t0.elapsed().as_secs_f64(); + + // CPU BLAS3. + let mut cpu = mm.clone(); + let t1 = Instant::now(); + let cpu_rank = cpu.row_reduce_blas3(); + let cpu_secs = t1.elapsed().as_secs_f64(); + + // Correctness: materialize device RREF and compare to CPU. + let stride = n.div_ceil(64); + let dev_limbs = gpu.download(&dm)?; + let perm_host = gpu.download_u32(&perm)?; + let mut e_limbs = vec![0u64; n * stride]; + for k in 0..r { + let src = perm_host[k] as usize * stride; + e_limbs[k * stride..k * stride + stride].copy_from_slice(&dev_limbs[src..src + stride]); + } + let e = Matrix::from_data(TWO, n, n, e_limbs); + let ok = r == cpu_rank && e == cpu; + + println!( + " n={n:5} rank={r:5}: device {dev_secs:7.3}s cpu-blas3 {cpu_secs:7.3}s speedup \ + {:5.2}x [{}]", + cpu_secs / dev_secs, + if ok { "correct" } else { "WRONG" } + ); + if !ok { + std::process::exit(1); + } + } + Ok(()) +} diff --git a/ext/crates/fp-cuda/examples/row_reduce_demo.rs b/ext/crates/fp-cuda/examples/row_reduce_demo.rs new file mode 100644 index 0000000000..99a60f621d --- /dev/null +++ b/ext/crates/fp-cuda/examples/row_reduce_demo.rs @@ -0,0 +1,112 @@ +//! Phase 5 gate (BLAS3-GPU-HANDOFF §5, the decisive one): the full device +//! reduction `row_reduce_dev` (forward pass + back-substitution over one +//! persistent buffer) must equal `fp::Matrix::row_reduce` bit-for-bit — the RREF +//! matrix *and* the pivot list. +//! +//! The device leaves the RREF pivot rows at perm positions [0, r) in ascending +//! column order; we materialize them into the canonical layout (pivot k at row +//! k, zero rows below) and compare to the CPU reducer's output. Also checks +//! agreement with `row_reduce_blas3`, the CPU BLAS3 oracle this port mirrors. +//! +//! Run with `cargo run -p fp-cuda --example row_reduce_demo`. + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::upload_matrix; +use rand::Rng; + +fn pivot_columns(m: &Matrix) -> Vec { + (0..m.columns()).filter(|&q| m.pivots()[q] >= 0).collect() +} + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + println!("=== fp-cuda row_reduce_dev (Phase 5, full RREF) demo ==="); + + let mut rng = rand::rng(); + let mut any_fail = false; + // (rows, cols, rank): rank=0 → full-rank random; rank>0 → rank-deficient. + let cases: &[(usize, usize, usize)] = &[ + (1, 1, 0), + (64, 64, 0), + (100, 130, 0), + (200, 200, 0), + (130, 130, 40), + (300, 130, 30), + (500, 320, 50), + (1000, 256, 0), + (1000, 500, 100), + (2000, 640, 200), + (64, 300, 0), + (777, 333, 111), + ]; + for &(rows, cols, rank) in cases { + let mm = if rank == 0 { + let v: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) + } else { + let a: Vec> = (0..rows) + .map(|_| (0..rank).map(|_| rng.random::() as u32).collect()) + .collect(); + let b: Vec> = (0..rank) + .map(|_| (0..cols).map(|_| rng.random::() as u32).collect()) + .collect(); + &Matrix::from_vec(TWO, &a) * &Matrix::from_vec(TWO, &b) + }; + + // CPU oracles. + let mut reference = mm.clone(); + let ref_rank = reference.row_reduce(); + let ref_pivots = pivot_columns(&reference); + let mut blas3 = mm.clone(); + blas3.row_reduce_blas3(); + + // Device full reduction. + let mut dm = upload_matrix(&gpu, &mm)?; + let (perm, r, mut pivcols) = gpu.row_reduce_dev(&mut dm)?; + let dev_limbs = gpu.download(&dm)?; + let perm_host = gpu.download_u32(&perm)?; + + // Materialize the canonical RREF: pivot k at row k (perm[k]), zeros below. + let stride = cols.div_ceil(64); + let mut e_limbs = vec![0u64; rows * stride]; + for k in 0..r { + let src = perm_host[k] as usize * stride; + e_limbs[k * stride..k * stride + stride].copy_from_slice(&dev_limbs[src..src + stride]); + } + let e = Matrix::from_data(TWO, rows, cols, e_limbs); + + pivcols.sort_unstable(); + let ok_rank = r == ref_rank; + let ok_piv = pivcols == ref_pivots; + let ok_rref = e == reference; + let ok_blas3 = e == blas3; + let ok = ok_rank && ok_piv && ok_rref && ok_blas3; + println!( + " rows={rows} cols={cols} rank={ref_rank}: {} (rref {} vs-blas3 {} rank {} piv {})", + if ok { "OK" } else { "MISMATCH" }, + yn(ok_rref), + yn(ok_blas3), + yn(ok_rank), + yn(ok_piv), + ); + if !ok { + any_fail = true; + } + } + + if any_fail { + println!("Some cases mismatched."); + std::process::exit(1); + } + println!("All cases matched (device row_reduce_dev == CPU row_reduce, bit-exact)."); + Ok(()) +} + +fn yn(b: bool) -> &'static str { + if b { "ok" } else { "BAD" } +} diff --git a/ext/crates/fp-cuda/examples/tune.rs b/ext/crates/fp-cuda/examples/tune.rs new file mode 100644 index 0000000000..f0125e983a --- /dev/null +++ b/ext/crates/fp-cuda/examples/tune.rs @@ -0,0 +1,55 @@ +//! Fast tuning target: kernel-only TOPS at a couple of sizes, no CPU +//! cross-check, single correctness spot-check at 4096. Used by the sweep +//! driver to compare tuning-knob configurations quickly. +//! +//! Run: `cargo run --release -p fp-cuda --example tune` + +use fp::{matrix::Matrix, prime::TWO}; +use fp_cuda::GpuContext; + +mod common; +use common::{matmul_b1, matmul_b1_timed}; +use rand::Rng; + +fn binary_tops(m: usize, k: usize, n: usize, secs: f64) -> f64 { + 2.0 * (m as f64) * (n as f64) * (k as f64) / secs / 1e12 +} + +fn main() -> Result<(), Box> { + let gpu = GpuContext::new(0)?; + let mut rng = rand::rng(); + let mut make = |rows: usize, cols: usize| { + let data: Vec = (0..rows * cols.div_ceil(64)) + .map(|_| rng.random()) + .collect(); + Matrix::from_data(TWO, rows, cols, data) + }; + + // One cheap correctness spot-check so a broken config is caught fast. + { + let a = make(4096, 4096); + let b = make(4096, 4096); + let cpu = &a * &b; + let gpu_ref = matmul_b1(&gpu, &a, &b)?; + if cpu != gpu_ref { + eprintln!("CORRECTNESS FAILURE at 4096"); + std::process::exit(1); + } + } + + for &(m, k, n, iters) in &[ + (8192usize, 8192, 8192, 30), + (16384, 16384, 16384, 15), + (32768, 32768, 32768, 8), + ] { + let a = make(m, k); + let b = make(k, n); + let (_, secs) = matmul_b1_timed(&gpu, &a, &b, iters)?; + println!( + " {m:>6} x {k:>6} x {n:>6}: {:>7.1} TOPS ({:>8.3} ms)", + binary_tops(m, k, n, secs), + secs * 1e3, + ); + } + Ok(()) +} diff --git a/ext/crates/fp-cuda/src/lib.rs b/ext/crates/fp-cuda/src/lib.rs new file mode 100644 index 0000000000..9a574ddcc9 --- /dev/null +++ b/ext/crates/fp-cuda/src/lib.rs @@ -0,0 +1,1655 @@ +//! CUDA backend for `fp::blas` F_2 matrix multiplication on Hopper. +//! +//! Both operands are pre-arranged on the host as plain row-major K-major tiles +//! and loaded via TMA with 128B swizzle, which lands them in the SMEM layout the +//! swizzled wgmma matrix descriptors expect. The kernel register-blocks a +//! TILE_M×(NG*64) output tile per CTA out of MSTRIPS m64n128 wgmma.b1 strips +//! that share each loaded B tile (cuts operand-refill bandwidth, the bottleneck). + +use std::{ffi::c_void, mem::MaybeUninit, sync::Arc, time::Instant}; + +use cudarc::{ + driver::{ + CudaContext, CudaFunction, CudaModule, CudaSlice, CudaStream, DevicePtr, DeviceRepr, + LaunchConfig, PushKernelArg, sys, + }, + nvrtc::Ptx, +}; + +static PTX_IMAGE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/matmul_b1.ptx")); + +const TILE_M: usize = 192; // MW*MSTRIPS in the kernel; must match +const TILE_K: usize = 1024; +const KL: usize = TILE_K / 64; // 16 +const THREADS: u32 = 256; // 2 warpgroups: producer (0..128) + consumer (128..256) +const NG: u32 = 2; // output column-limbs per CTA (NB/64 = 128/64); must match the kernel +const STAGES: usize = 4; // K-loop pipeline depth; must match the kernel +const CLUSTER: usize = 2; // CTAs per cluster along M (multicast B); must match the kernel + +/// Adaptive forward-pass panel width in limbs (b = 64·bl columns). Wider panels +/// raise the trailing GEMM's contraction dimension toward b, reclaiming the +/// K-padding waste (Loss 1). The counter-pressure is the promotion cost: with the +/// single-CTA promote_pivots it is O(bl) total, so narrow panels win; but the +/// cooperative promote_coop (used at stride ≥ 1024) is ~bl-independent, so there +/// the panel can be widened until the forward GEMM stops padding (bl=16 ⇒ K=1024 +/// exactly). Measured optima (half-rank, H200): bl=2 @ n=2¹⁵ (stride 512, single- +/// CTA), 8 @ 2¹⁶ (1024), 16 @ 2¹⁷ (2048) — i.e. stride/256 without the coop +/// promote, stride/128 with it. Capped at 16. Override with `FP_CUDA_BL`. +fn adaptive_bl(stride: usize) -> usize { + let div = if stride >= 1024 { 128 } else { 256 }; + (stride / div).clamp(1, 16) +} + +/// Lets us pass a `CUtensorMap` by value as a (grid-constant) kernel argument +/// through cudarc's typed launch builder. `repr(transparent)` so the pointer +/// cudarc pushes is the address of the 128-byte descriptor itself. +#[repr(transparent)] +struct TmaArg(sys::CUtensorMap); +unsafe impl DeviceRepr for TmaArg {} + +/// A bit-packed F₂ matrix resident in device memory, in the natural row-major, +/// K-major limb layout `fp::Matrix` uses: `stride = cols.div_ceil(64)` u64 per +/// row, `rows * stride` u64 total, one bit per entry, bits past `cols` zero. +/// +/// This is the persistent buffer the row-reduction port operates over: uploaded +/// once, mutated in place by device kernels, downloaded once (design §6). It is +/// `fp`-agnostic (raw limbs) so `fp-cuda` stays free of a dependency on `fp`. +pub struct DeviceMatrix { + pub buf: CudaSlice, + pub rows: usize, + pub cols: usize, + pub stride: usize, +} + +impl DeviceMatrix { + pub fn stride(&self) -> usize { + self.stride + } +} + +pub struct GpuContext { + ctx: Arc, + #[allow(dead_code)] + module: Arc, + kernel: CudaFunction, + // Device-resident packing/epilogue kernels for the row-reduction port. + pack_a: CudaFunction, + pack_b: CudaFunction, + xor_into: CudaFunction, + panel_factor: CudaFunction, + panel_factor_coop: CudaFunction, + mark_live: CudaFunction, + promote_pivots: CudaFunction, + promote_coop: CudaFunction, + zero_pivot_l: CudaFunction, + gather_rows: CudaFunction, + block_reduce_rref: CudaFunction, + block_reduce_coop: CudaFunction, + gather_cols: CudaFunction, + xor_into_perm: CudaFunction, +} + +impl GpuContext { + pub fn new(device_id: usize) -> Result> { + let ctx = CudaContext::new(device_id)?; + let ptx = Ptx::from_src(String::from_utf8(PTX_IMAGE.to_vec())?); + let module = ctx.load_module(ptx)?; + let kernel = module.load_function("matmul_b1_kernel")?; + let pack_a = module.load_function("pack_a")?; + let pack_b = module.load_function("pack_b")?; + let xor_into = module.load_function("xor_into")?; + let panel_factor = module.load_function("panel_factor")?; + let panel_factor_coop = module.load_function("panel_factor_coop")?; + let mark_live = module.load_function("mark_live")?; + let promote_pivots = module.load_function("promote_pivots")?; + let promote_coop = module.load_function("promote_coop")?; + let zero_pivot_l = module.load_function("zero_pivot_l")?; + let gather_rows = module.load_function("gather_rows")?; + let block_reduce_rref = module.load_function("block_reduce_rref")?; + let block_reduce_coop = module.load_function("block_reduce_coop")?; + let gather_cols = module.load_function("gather_cols")?; + let xor_into_perm = module.load_function("xor_into_perm")?; + Ok(Self { + ctx, + module, + kernel, + pack_a, + pack_b, + xor_into, + panel_factor, + panel_factor_coop, + mark_live, + promote_pivots, + promote_coop, + zero_pivot_l, + gather_rows, + block_reduce_rref, + block_reduce_coop, + gather_cols, + xor_into_perm, + }) + } + + pub fn compute_capability(&self) -> Result<(i32, i32), Box> { + let major = self.ctx.attribute( + sys::CUdevice_attribute_enum::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, + )?; + let minor = self.ctx.attribute( + sys::CUdevice_attribute_enum::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, + )?; + Ok((major, minor)) + } + + pub fn default_stream(&self) -> Arc { + self.ctx.default_stream() + } + + pub fn kernel(&self) -> &CudaFunction { + &self.kernel + } +} + +/// Multiply two bit-packed F₂ matrices on the GPU. +/// +/// Operands are plain **row-major, K-major** limb arrays — the exact layout +/// `fp::Matrix::to_bytes` produces (little-endian `u64` limbs, one bit per +/// entry, `columns.div_ceil(64)` limbs per row, no inter-row padding): +/// +/// - `a`: the `m`×`k` left operand, `m * k.div_ceil(64)` limbs. +/// - `b`: the `k`×`n` right operand, `k * n.div_ceil(64)` limbs. +/// +/// Returns C = A·B as `m * n.div_ceil(64)` limbs in the same layout, ready to +/// hand to `fp::Matrix::from_data`. This keeps `fp-cuda` free of any dependency +/// on `fp` (the higher-level crate depends on this one, not the reverse). +pub fn matmul_b1_raw( + gpu: &GpuContext, + a: &[u64], + m: usize, + k: usize, + b: &[u64], + n: usize, +) -> Result, Box> { + Ok(matmul_b1_inner(gpu, a, m, k, b, n, 1)?.0) +} + +/// Like [`matmul_b1_raw`], but also returns the average **kernel-only** wall +/// time (seconds) over `time_iters` back-to-back launches, excluding host +/// (de)serialization, the TMA-layout pre-arrangement, and the H2D/D2H copies. +/// +/// The kernel zeroes its SMEM accumulator and writes C with a bulk-tensor +/// *store* (overwrite, not accumulate), so repeated launches against the same +/// device buffers are idempotent and the returned limbs are the correct +/// product. Use this to compare against the ~100-binary-TOPS pre-swizzle +/// kernel baseline; the end-to-end `cargo bench` figures are dominated by host +/// serialization and understate kernel throughput. +pub fn matmul_b1_raw_timed( + gpu: &GpuContext, + a: &[u64], + m: usize, + k: usize, + b: &[u64], + n: usize, + time_iters: usize, +) -> Result<(Vec, f64), Box> { + matmul_b1_inner(gpu, a, m, k, b, n, time_iters.max(1)) +} + +#[allow(clippy::too_many_arguments)] +fn matmul_b1_inner( + gpu: &GpuContext, + a: &[u64], + m: usize, + k: usize, + b: &[u64], + n: usize, + time_iters: usize, +) -> Result<(Vec, f64), Box> { + let n_lim = n.div_ceil(64); + assert_eq!(a.len(), m * k.div_ceil(64), "A limb count mismatch"); + assert_eq!(b.len(), k * n_lim, "B limb count mismatch"); + + let k_padded = k.next_multiple_of(TILE_K); + // Pad M to a whole number of clusters (CLUSTER M-tiles) so every cluster + // rank has a valid M-tile; the extra padded rows produce zeros that the + // `take(m)` readback trims. + let m_padded = m.next_multiple_of(TILE_M * CLUSTER); + // Each CTA computes a TILE_M×(NG*64) output block via MSTRIPS m64n128 wgmmas, + // so B (and the C output) are grouped/padded to whole NG-limb column tiles. + let n_groups = n_lim.div_ceil(NG as usize); + let n_padded_lim = n_groups * NG as usize; + + let stream = gpu.ctx.default_stream(); + + let a_padded = pad_2d(a, m, k.div_ceil(64), m_padded, k_padded / 64); + let b_padded = pad_2d(b, k, n_lim, k_padded, n_lim); + + // Gather A into row-major K-major tiles; the TMA applies the 128B swizzle. + let a_interleaved = interleave_a(&a_padded, m_padded, k_padded); + // Pre-transpose B into row-major K-major tiles (swizzled by the TMA). + let bt = transpose_b(&b_padded, k_padded, n_lim); + + let a_dev = stream.clone_htod(&a_interleaved)?; + let bt_dev = stream.clone_htod(&bt)?; + let c_dev = stream.alloc_zeros::(m_padded * n_padded_lim)?; + + let kernel_secs = run_gemm_kernel( + gpu, + &stream, + &a_dev, + &bt_dev, + &c_dev, + m_padded, + k_padded, + n_groups, + n_padded_lim, + time_iters, + )?; + + let c_all = stream.clone_dtoh(&c_dev)?; + let c_limbs: Vec = c_all + .chunks_exact(n_padded_lim) + .take(m) + .flat_map(|row| row[..n_lim].iter().copied()) + .collect(); + Ok((c_limbs, kernel_secs)) +} + +/// Encode the TMA descriptors and launch the `wgmma.b1` GEMM over operands that +/// are **already interleaved/transposed and resident on device**. Shared by the +/// host round-trip path ([`matmul_b1_inner`]) and the device-resident path +/// ([`GpuContext::matmul_b1_dev`]) — the only difference between them is where +/// the packed operands come from and whether `C` is downloaded. Returns the +/// average kernel-only wall time over `time_iters` launches. +#[allow(clippy::too_many_arguments)] +fn run_gemm_kernel( + gpu: &GpuContext, + stream: &Arc, + a_dev: &cudarc::driver::CudaSlice, + bt_dev: &cudarc::driver::CudaSlice, + c_dev: &cudarc::driver::CudaSlice, + m_padded: usize, + k_padded: usize, + n_groups: usize, + n_padded_lim: usize, + time_iters: usize, +) -> Result> { + let m_tiles = m_padded / TILE_M; + let k_chunks = k_padded / TILE_K; + + // Raw device addresses for the TMA descriptors. The returned guards keep the + // reads ordered on the stream; hold them until after the launch. + let (a_ptr, _ga) = a_dev.device_ptr(stream); + let (b_ptr, _gb) = bt_dev.device_ptr(stream); + let (c_ptr, _gc) = c_dev.device_ptr(stream); + + // TMA tensor maps. A: TILE_M-row block per (k_chunk, M-block), split into + // MSTRIPS m64 strips by the consumer. B: (NG*64)-column tile per (k_chunk, + // column group), reused across the strips. Both have a 128-byte inner dim + // (= the 128B swizzle width). C: TILE_M-row × NG-limb output blocks, no + // swizzle, for the bulk store. + let tma_a = encode_tma( + a_ptr, + [32, (k_chunks * m_tiles * TILE_M) as u64], + [32, TILE_M as u32], + 128, + sys::CUtensorMapSwizzle_enum::CU_TENSOR_MAP_SWIZZLE_128B, + )?; + let tma_b = encode_tma( + b_ptr, + [32, (k_chunks * n_groups * NG as usize * 64) as u64], + [32, (NG as usize * 64) as u32], + 128, + sys::CUtensorMapSwizzle_enum::CU_TENSOR_MAP_SWIZZLE_128B, + )?; + let tma_c = encode_tma( + c_ptr, + [(n_padded_lim * 2) as u64, m_padded as u64], + [(NG as usize * 2) as u32, TILE_M as u32], + (n_padded_lim * 8) as u64, + sys::CUtensorMapSwizzle_enum::CU_TENSOR_MAP_SWIZZLE_NONE, + )?; + + // Dynamic SMEM per CTA: sA + sB + 2*sC (double-buffered) + 2*STAGES mbarriers. + let tile_a = TILE_M * KL; // TILE_M-row A block + let tile_b = NG as usize * 64 * KL; // (NG*64)-col B tile + let smem_u64 = STAGES * tile_a + STAGES * tile_b + 2 * NG as usize * TILE_M + 2 * STAGES; + let smem_bytes = (smem_u64 * std::mem::size_of::()) as u32; + + // Opt in to >48 KB shared memory (Hopper static default cap). + gpu.kernel.set_attribute( + sys::CUfunction_attribute_enum::CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, + smem_bytes as i32, + )?; + + // Persistent grid: co-resident CTAs = (occupancy per SM) × SM count, so the + // grid exactly fills the machine and the kernel's persistent loop sweeps all + // output tiles in grouped-rasterized order. Rounded to a whole number of + // clusters (`__cluster_dims__` requires gridDim.x % CLUSTER == 0); surplus + // clusters run an empty loop. + // + // This is 1 CTA/SM at the register-blocked config, and that is optimal: + // 2 CTAs/SM was measured (2026-07-07 H200) and loses badly. Two CTAs need the + // compiled register count ≤128/thread (2·256·128 = the 64K reg file), but the + // resident accumulator that gives the kernel its arithmetic intensity is + // 192 regs/thread at MSTRIPS=3. Shrinking it to fit two CTAs (MSTRIPS=1, + // 64-reg acc → occ=2) collapses AI and drops 16384 from ~8,600 to ~5,500 + // TOPS. High AI (big accumulator) and high occupancy compete for the same + // register file and AI wins, so we run 1 CTA/SM with the largest accumulator + // that fits under the 255-reg cap. + let sms = gpu + .ctx + .attribute(sys::CUdevice_attribute_enum::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)? + as u32; + let occ = gpu + .kernel + .occupancy_max_active_blocks_per_multiprocessor(THREADS, smem_bytes as usize, None)? + .max(1); + let mut num_ctas = (occ * sms / CLUSTER as u32).max(1) * CLUSTER as u32; + // Diagnostic: cap the persistent grid to probe how much of a small GEMM's + // time is the persistent-grid startup (cluster sync + mbar init + pipeline + // fill across occ×SMs CTAs). The persistent loop handles any multiple of + // CLUSTER; fewer CTAs just do more tile-iterations each. + if let Some(cap) = std::env::var("FP_CUDA_GEMM_CTAS") + .ok() + .and_then(|v| v.parse::().ok()) + { + num_ctas = (cap / CLUSTER as u32).max(1) * CLUSTER as u32; + } + + let ta = TmaArg(tma_a); + let tb = TmaArg(tma_b); + let tc = TmaArg(tma_c); + let mt = m_tiles as u32; + let ng = n_groups as u32; + let m_val = m_padded as u32; + let k_val = k_padded as u32; + + let launch = || -> Result<(), cudarc::driver::DriverError> { + let cfg = LaunchConfig { + grid_dim: (num_ctas, 1, 1), + block_dim: (THREADS, 1, 1), + shared_mem_bytes: smem_bytes, + }; + let mut lb = stream.launch_builder(&gpu.kernel); + lb.arg(&ta) + .arg(&tb) + .arg(&tc) + .arg(&mt) + .arg(&ng) + .arg(&m_val) + .arg(&k_val); + unsafe { lb.launch(cfg) }?; + Ok(()) + }; + + // Warm up once (untimed) when measuring, so the timed loop excludes any + // first-launch JIT/allocation costs. + if time_iters > 1 { + launch()?; + stream.synchronize()?; + } + + let start = Instant::now(); + for _ in 0..time_iters { + launch()?; + } + stream.synchronize()?; + let kernel_secs = start.elapsed().as_secs_f64() / time_iters as f64; + + Ok(kernel_secs) +} + +/// A 1D launch config: `ceil(total / 256)` blocks of 256 threads. +fn cfg_1d(total: usize) -> LaunchConfig { + const T: u32 = 256; + let blocks = total.div_ceil(T as usize).max(1) as u32; + LaunchConfig { + grid_dim: (blocks, 1, 1), + block_dim: (T, 1, 1), + shared_mem_bytes: 0, + } +} + +impl GpuContext { + /// Device-resident F₂ GEMM. Both operands live on device in the **natural** + /// row-major, K-major limb layout that `fp::Matrix` stores (`a_dev`: `m × + /// k.div_ceil(64)` u64; `b_dev`: `k × n.div_ceil(64)` u64). The pack into the + /// wgmma tile layout (interleave A, bit-transpose B) runs on device, so there + /// is no host round-trip — this is the persistent-buffer primitive the + /// row-reduction port needs (design §6). + /// + /// Returns `C = A·B` as a **padded** device buffer of `m_padded × + /// n_padded_lim` u64 (valid data in the first `m` rows and first + /// `n.div_ceil(64)` limbs of each row), plus its row stride `n_padded_lim`. + /// The padded stride is what [`GpuContext::xor_into_region`] consumes; a + /// standalone caller trims it on readback. + pub fn matmul_b1_dev( + &self, + a_dev: &CudaSlice, + m: usize, + k: usize, + b_dev: &CudaSlice, + n: usize, + ) -> Result<(CudaSlice, usize), Box> { + let sa = k.div_ceil(64); + self.matmul_b1_dev_strided(a_dev, m, k, sa, b_dev, n) + } + + /// Like [`matmul_b1_dev`](Self::matmul_b1_dev) but A may be a row-sub-block of + /// a wider device buffer: `a_row_stride` is A's actual per-row limb stride + /// (≥ `k.div_ceil(64)`), while only the first `k.div_ceil(64)` limbs of each + /// row are the operand. Used by the wide-panel forward update, whose + /// multiplier matrix L is stored with stride `bl` but has only `ceil(pr/64)` + /// occupied limbs. + pub fn matmul_b1_dev_strided( + &self, + a_dev: &CudaSlice, + m: usize, + k: usize, + a_row_stride: usize, + b_dev: &CudaSlice, + n: usize, + ) -> Result<(CudaSlice, usize), Box> { + let sa = k.div_ceil(64); + let n_lim = n.div_ceil(64); + assert!(a_row_stride >= sa, "a_row_stride must cover k limbs"); + assert_eq!(a_dev.len(), m * a_row_stride, "A limb count mismatch"); + assert_eq!(b_dev.len(), k * n_lim, "B limb count mismatch"); + + let k_padded = k.next_multiple_of(TILE_K); + let m_padded = m.next_multiple_of(TILE_M * CLUSTER); + let m_tiles = m_padded / TILE_M; + let k_chunks = k_padded / TILE_K; + let n_groups = n_lim.div_ceil(NG as usize); + let n_padded_lim = n_groups * NG as usize; + + let stream = self.ctx.default_stream(); + + // Pack A → interleaved row-major K-major tiles (m_padded × k_padded/64). + // pack_a/pack_b/the GEMM fully overwrite these buffers (padding written as + // explicit zeros), so allocate uninitialized and skip the memset — the + // per-call zeroing of the multi-GB C output is pure waste at large n. + let a_int_len = m_padded * (k_padded / 64); + let a_int = unsafe { stream.alloc::(a_int_len) }?; + { + let (m_orig, sa_orig, a_str, mt, total) = ( + m as u32, + sa as u32, + a_row_stride as u32, + m_tiles as u32, + a_int_len as u32, + ); + let mut lb = stream.launch_builder(&self.pack_a); + lb.arg(&a_int) + .arg(a_dev) + .arg(&m_orig) + .arg(&sa_orig) + .arg(&a_str) + .arg(&mt) + .arg(&total); + unsafe { lb.launch(cfg_1d(a_int_len)) }?; + } + + // Pack B → bit-transposed K-major tiles. + let bt_len = k_chunks * n_groups * (NG as usize * 64 * KL); + let bt = unsafe { stream.alloc::(bt_len) }?; + { + let (k_orig, nl, ng, total) = (k as u32, n_lim as u32, n_groups as u32, bt_len as u32); + let mut lb = stream.launch_builder(&self.pack_b); + lb.arg(&bt) + .arg(b_dev) + .arg(&k_orig) + .arg(&nl) + .arg(&ng) + .arg(&total); + unsafe { lb.launch(cfg_1d(bt_len)) }?; + } + + // The GEMM writes C with a bulk-tensor store (overwrite, not accumulate), + // covering every output tile; xor_into only reads the first m rows and + // trailing_limbs columns, all written. So C needs no pre-zeroing. + let c_dev = unsafe { stream.alloc::(m_padded * n_padded_lim) }?; + run_gemm_kernel( + self, + &stream, + &a_int, + &bt, + &c_dev, + m_padded, + k_padded, + n_groups, + n_padded_lim, + 1, + )?; + + Ok((c_dev, n_padded_lim)) + } + + /// XOR a padded GEMM output `c_dev` (`m × c_stride` u64/row, as returned by + /// [`GpuContext::matmul_b1_dev`]) into a region of a persistent device matrix + /// `dst`: `dst[j][dst_limb + col] ^= c_dev[j][col]` for `j < m`, `col < + /// width`. This is the fused trailing-update / back-sub epilogue (design + /// §4.4/§4.6): the `M[region] ^= L·U` update with no fresh C alloc + D2H. + #[allow(clippy::too_many_arguments)] + pub fn xor_into_region( + &self, + stream: &Arc, + dst: &mut CudaSlice, + c_dev: &CudaSlice, + m: usize, + width: usize, + dst_stride: usize, + dst_limb: usize, + c_stride: usize, + ) -> Result<(), Box> { + let total = m * width; + let (mm, w, ds, dl, cs) = ( + m as u32, + width as u32, + dst_stride as u32, + dst_limb as u32, + c_stride as u32, + ); + let mut lb = stream.launch_builder(&self.xor_into); + lb.arg(dst) + .arg(c_dev) + .arg(&mm) + .arg(&w) + .arg(&ds) + .arg(&dl) + .arg(&cs); + unsafe { lb.launch(cfg_1d(total)) }?; + Ok(()) + } + + /// Convenience wrapper: upload two host operands, run [`matmul_b1_dev`], and + /// download the trimmed `m × n.div_ceil(64)` product. Same signature and + /// result as [`matmul_b1_raw`] but exercising the device-resident packing + + /// GEMM path end to end — used to validate that path bit-for-bit against the + /// host oracle. + /// + /// [`matmul_b1_dev`]: GpuContext::matmul_b1_dev + pub fn matmul_b1_dev_roundtrip( + &self, + a: &[u64], + m: usize, + k: usize, + b: &[u64], + n: usize, + ) -> Result, Box> { + let n_lim = n.div_ceil(64); + let stream = self.ctx.default_stream(); + let a_dev = stream.clone_htod(a)?; + let b_dev = stream.clone_htod(b)?; + let (c_dev, n_padded_lim) = self.matmul_b1_dev(&a_dev, m, k, &b_dev, n)?; + let c_all = stream.clone_dtoh(&c_dev)?; + Ok(c_all + .chunks_exact(n_padded_lim) + .take(m) + .flat_map(|row| row[..n_lim].iter().copied()) + .collect()) + } + + /// Upload a bit-packed F₂ matrix (natural row-major limb layout, `rows * + /// cols.div_ceil(64)` u64) to device memory. One H2D copy. + pub fn upload( + &self, + data: &[u64], + rows: usize, + cols: usize, + ) -> Result> { + let stride = cols.div_ceil(64); + assert_eq!(data.len(), rows * stride, "limb count mismatch"); + let buf = self.ctx.default_stream().clone_htod(data)?; + Ok(DeviceMatrix { + buf, + rows, + cols, + stride, + }) + } + + /// Download a [`DeviceMatrix`] back to host limbs (natural layout). One D2H. + pub fn download(&self, dm: &DeviceMatrix) -> Result, Box> { + Ok(self.ctx.default_stream().clone_dtoh(&dm.buf)?) + } + + /// Download a device `u32` buffer (e.g. a `perm` vector) to host. + pub fn download_u32(&self, s: &CudaSlice) -> Result, Box> { + Ok(self.ctx.default_stream().clone_dtoh(s)?) + } + + /// The fused trailing-update / back-substitution epilogue over persistent + /// device buffers (design §4.4/§4.6): `dst[:, col_off:] ^= L · U`, in place, + /// where `L` is `dst.rows × k` and `U` is `k × t` (`k = l.cols = u.rows`, + /// `t = u.cols`). `col_off` must be a limb boundary (multiple of 64) and the + /// trailing region `[col_off, dst.cols)` must be exactly `t` columns wide — + /// i.e. `col_off + t == dst.cols` — so the whole-limb XOR lands correctly. + /// + /// Runs [`matmul_b1_dev`](Self::matmul_b1_dev) into a scratch device C and + /// XORs it into `dst` with [`xor_into_region`](Self::xor_into_region) — no + /// host round-trip, no D2H of the product. + pub fn gemm_xor_into( + &self, + dst: &mut DeviceMatrix, + l: &DeviceMatrix, + u: &DeviceMatrix, + col_off: usize, + ) -> Result<(), Box> { + assert_eq!(col_off % 64, 0, "col_off must be a limb boundary"); + assert_eq!(l.rows, dst.rows, "L rows must match dst rows"); + assert_eq!( + l.cols, u.rows, + "inner dimension mismatch (L.cols != U.rows)" + ); + assert_eq!( + col_off + u.cols, + dst.cols, + "trailing region must be U.cols wide" + ); + let (m, k, t) = (dst.rows, l.cols, u.cols); + if m == 0 || k == 0 || t == 0 { + return Ok(()); + } + let (c_dev, _n_padded_lim) = self.matmul_b1_dev(&l.buf, m, k, &u.buf, t)?; + let width = t.div_ceil(64); // == dst.stride - col_off/64 + let stream = self.ctx.default_stream(); + self.xor_into_region( + &stream, + &mut dst.buf, + &c_dev, + m, + width, + dst.stride, + col_off / 64, + _n_padded_lim, + )?; + stream.synchronize()?; + Ok(()) + } + + /// Allocate the identity virtual-row permutation `perm = [0, 1, …, m-1]` on + /// device (design §4.3). Kernels dereference rows as `M[perm[i]]`; row swaps + /// are `perm` swaps, so the matrix bytes never move. + pub fn identity_perm(&self, m: usize) -> Result, Box> { + let host: Vec = (0..m as u32).collect(); + Ok(self.ctx.default_stream().clone_htod(&host)?) + } + + /// Factor one 64-bit column panel (limb `plimb`) in place over the + /// persistent buffer, forward-only, starting from pivot row `r` (design §5, + /// the b=64 base kernel). Rows are addressed through `perm`; a pivot is + /// promoted by swapping its `perm` entry to position `r + pivot_index`. The + /// multiplier bits captured while clearing rows *below* each pivot are ORed + /// into `l` (indexed by original row id, so `l` needs no permutation). + /// + /// `l` must be an `m × (≥64-column)` device matrix (one limb per row is + /// enough since a panel yields ≤ 64 pivots); the caller zeroes the panel's + /// pivot-index bits before the call. Returns `(pr, pivcols)` — the pivots + /// found and their absolute columns — read back to host (a few hundred + /// bytes); everything else stays on device. + pub fn panel_factor( + &self, + m: &mut DeviceMatrix, + perm: &mut CudaSlice, + l: &mut DeviceMatrix, + plimb: usize, + r: usize, + ) -> Result<(usize, Vec), Box> { + assert_eq!(perm.len(), m.rows, "perm length must equal rows"); + assert_eq!(l.rows, m.rows, "L rows must equal M rows"); + let stream = self.ctx.default_stream(); + + const THREADS: u32 = 256; + let pivcols = stream.alloc_zeros::(64)?; + let pr_out = stream.alloc_zeros::(1)?; + + let (plimb_u, r_u, n_u, m_u, stride_u, l_stride_u) = ( + plimb as u32, + r as u32, + m.cols as u32, + m.rows as u32, + m.stride as u32, + l.stride as u32, + ); + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (THREADS, 1, 1), + shared_mem_bytes: THREADS * std::mem::size_of::() as u32, + }; + let mut lb = stream.launch_builder(&self.panel_factor); + lb.arg(&mut m.buf) + .arg(perm) + .arg(&mut l.buf) + .arg(&pivcols) + .arg(&pr_out) + .arg(&plimb_u) + .arg(&r_u) + .arg(&n_u) + .arg(&m_u) + .arg(&stride_u) + .arg(&l_stride_u); + unsafe { lb.launch(cfg) }?; + + let pr = stream.clone_dtoh(&pr_out)?[0] as usize; + let cols = stream.clone_dtoh(&pivcols)?; + Ok((pr, cols[..pr].to_vec())) + } + + /// Grid-parallel (cooperative) equivalent of [`panel_factor`](Self::panel_factor): + /// identical math, but each of the 64 sequential bit-steps spreads its + /// find-first and masked-XOR across the whole grid, with a self-contained + /// atomic grid barrier between steps. The single-CTA `panel_factor` uses one + /// SM of ~132 and dominates the forward pass (~76% of GPU time profiled); this + /// removes that bottleneck. Launched via `cuLaunchCooperativeKernel` with all + /// CTAs co-resident (required by the spin barrier). Same `(pr, pivcols)` + /// read-back. + pub fn panel_factor_coop( + &self, + m: &mut DeviceMatrix, + perm: &mut CudaSlice, + l: &mut DeviceMatrix, + ppanel: usize, + bl: usize, + r: usize, + m_active: usize, + ) -> Result<(usize, Vec), Box> { + assert_eq!(perm.len(), m.rows, "perm length must equal rows"); + assert_eq!(l.rows, m.rows, "L rows must equal M rows"); + assert!(l.stride >= bl, "L stride must be at least bl"); + assert!(m_active <= m.rows && m_active >= r, "m_active out of range"); + let stream = self.ctx.default_stream(); + + const THREADS: u32 = 256; + let smem = THREADS * std::mem::size_of::() as u32; + // Up to bl·64 pivots in a wide panel. + let pivcols = stream.alloc_zeros::(bl * 64)?; + let pr_out = stream.alloc_zeros::(1)?; + // scratch = [barrier(0), g_min, g_pr]; alloc_zeros gives barrier == 0. + let scratch = stream.alloc_zeros::(3)?; + let g_pivword = stream.alloc_zeros::(bl)?; + + // Co-resident grid: at most occ×SMs CTAs (cooperative launch cap), and no + // more than needed to cover the rows once. + let sms = self + .ctx + .attribute(sys::CUdevice_attribute_enum::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)? + as u32; + let occ = self + .panel_factor_coop + .occupancy_max_active_blocks_per_multiprocessor(THREADS, smem as usize, None)? + .max(1); + // Only rows [r, m_active) are scanned/updated (dead rows excluded), so + // size the grid and the kernel's row bound to m_active. + let rows_worth = (m_active as u32).div_ceil(THREADS).max(1); + let num_ctas = (occ * sms).min(rows_worth).max(1); + // panel_factor is partly grid-barrier-bound (a grid_sync per pivot bit), + // so a smaller grid gives cheaper barriers while still covering the panel + // work — measured +3% at 2^16. Cap at 128 (H200 sweet spot); overridable. + let num_ctas = std::env::var("FP_CUDA_PF_CTAS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(128) + .clamp(1, num_ctas); + + let (ppanel_u, bl_u, r_u, n_u, m_u, stride_u, l_stride_u, tc) = ( + ppanel as u32, + bl as u32, + r as u32, + m.cols as u32, + m_active as u32, + m.stride as u32, + l.stride as u32, + num_ctas, + ); + let cfg = LaunchConfig { + grid_dim: (num_ctas, 1, 1), + block_dim: (THREADS, 1, 1), + shared_mem_bytes: smem, + }; + let mut lb = stream.launch_builder(&self.panel_factor_coop); + lb.arg(&mut m.buf) + .arg(perm) + .arg(&mut l.buf) + .arg(&pivcols) + .arg(&pr_out) + .arg(&scratch) + .arg(&g_pivword) + .arg(&ppanel_u) + .arg(&bl_u) + .arg(&r_u) + .arg(&n_u) + .arg(&m_u) + .arg(&stride_u) + .arg(&l_stride_u) + .arg(&tc); + unsafe { lb.launch_cooperative(cfg) }?; + + let pr = stream.clone_dtoh(&pr_out)?[0] as usize; + let cols = stream.clone_dtoh(&pivcols)?; + Ok((pr, cols[..pr].to_vec())) + } + + /// Active-row compaction (design §8.2): mark the below rows [r, m_active) + /// that are entirely zero across the remaining columns [start_limb·64, n) — + /// permanently dead (they can never pivot and carry no multiplier) — and + /// stable-partition `perm[r..m_active]` so the live rows come first. Returns + /// the new active count `r + live`. The dead rows are parked in + /// [new_active, m_active) and never scanned again; pivot rows [0, r) are + /// untouched. The partition is done on the host (a few hundred KB of perm + + /// flags per call), which is negligible beside the panel work it saves. + fn compact_perm( + &self, + m: &DeviceMatrix, + perm: &mut CudaSlice, + r: usize, + m_active: usize, + start_limb: usize, + ) -> Result> { + if m_active <= r { + return Ok(m_active); + } + let stream = self.ctx.default_stream(); + let n_scan = m_active - r; + let live = unsafe { stream.alloc::(n_scan) }?; + { + let (r_u, m_u, sl, st) = ( + r as u32, + m_active as u32, + start_limb as u32, + m.stride as u32, + ); + let mut lb = stream.launch_builder(&self.mark_live); + lb.arg(&live) + .arg(&m.buf) + .arg(&*perm) + .arg(&r_u) + .arg(&m_u) + .arg(&sl) + .arg(&st); + unsafe { lb.launch(cfg_1d(n_scan)) }?; + } + let live_host = stream.clone_dtoh(&live)?; + let perm_host = stream.clone_dtoh(&perm.slice(r..m_active))?; + // Stable partition: live rows first (preserve order), dead rows after. + let mut ordered: Vec = Vec::with_capacity(n_scan); + for (i, &lv) in live_host.iter().enumerate() { + if lv != 0 { + ordered.push(perm_host[i]); + } + } + let new_active = r + ordered.len(); + for (i, &lv) in live_host.iter().enumerate() { + if lv == 0 { + ordered.push(perm_host[i]); + } + } + let mut view = perm.slice_mut(r..m_active); + stream.memcpy_htod(&ordered, &mut view)?; + Ok(new_active) + } + + /// Elementwise promote of the `pr` pivot rows (perm positions + /// `[r_piv, r_piv+pr)`): the cooperative forward-substitution replay of `L` + /// onto those rows' trailing. `l_limb_off` shifts the L read so bit `i` maps + /// to global multiplier bit `l_limb_off·64 + i` (0 for the whole-panel promote). + #[allow(clippy::too_many_arguments)] + fn promote_elem( + &self, + m: &mut DeviceMatrix, + perm: &CudaSlice, + l: &DeviceMatrix, + r_piv: usize, + pr: usize, + first_limb: usize, + trailing_limbs: usize, + l_limb_off: usize, + pc_barrier: &mut CudaSlice, + pc_cond: &CudaSlice, + pc_ctas: u32, + ) -> Result<(), Box> { + if pr == 0 || trailing_limbs == 0 { + return Ok(()); + } + let stream = self.ctx.default_stream(); + stream.memset_zeros(pc_barrier)?; + let (r_u, pr_u, fl, tl, st, ls, llo, tc) = ( + r_piv as u32, + pr as u32, + first_limb as u32, + trailing_limbs as u32, + m.stride as u32, + l.stride as u32, + l_limb_off as u32, + pc_ctas, + ); + let cfg = LaunchConfig { + grid_dim: (pc_ctas, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + }; + let mut lb = stream.launch_builder(&self.promote_coop); + lb.arg(&mut m.buf) + .arg(perm) + .arg(&l.buf) + .arg(&r_u) + .arg(&pr_u) + .arg(&fl) + .arg(&tl) + .arg(&st) + .arg(&ls) + .arg(&llo) + .arg(pc_barrier) + .arg(pc_cond) + .arg(&tc); + unsafe { lb.launch_cooperative(cfg) }?; + Ok(()) + } + + /// One trailing update `M[:, first_limb·64 : end_limb·64) ^= L · U` (design + /// §4.4): promote the `pr` pivot rows at perm positions `[r_piv, r_piv+pr)` + /// over the column range, drop them from `l`, gather them into `U`, run the + /// GEMM, and XOR the product into the region. Shared by the single-wide-panel + /// far update and the recursive intra-panel updates; `l` carries the pivots' + /// multipliers at bits `[0, pr)` (stride `l.stride`). `pc_*` are the + /// cooperative-promote scaffolding, created once by the caller. + #[allow(clippy::too_many_arguments)] + fn trailing_update( + &self, + m: &mut DeviceMatrix, + perm: &CudaSlice, + l: &mut DeviceMatrix, + r_piv: usize, + pr: usize, + first_limb: usize, + end_limb: usize, + promote_coop: bool, + pc_barrier: &mut CudaSlice, + pc_cond: &CudaSlice, + pc_ctas: u32, + ) -> Result<(), Box> { + let stream = self.ctx.default_stream(); + let (rows, stride, n) = (m.rows, m.stride, m.cols); + let trailing_limbs = end_limb - first_limb; + if pr == 0 || trailing_limbs == 0 { + return Ok(()); + } + let t = (end_limb * 64).min(n) - first_limb * 64; + + // (1) promote pivot rows' trailing, then (2) zero their L rows. + { + let (r_u, pr_u, fl, tl, st, ls) = ( + r_piv as u32, + pr as u32, + first_limb as u32, + trailing_limbs as u32, + stride as u32, + l.stride as u32, + ); + if promote_coop { + self.promote_elem( + m, + perm, + l, + r_piv, + pr, + first_limb, + trailing_limbs, + 0, + pc_barrier, + pc_cond, + pc_ctas, + )?; + } else { + let mut lb = stream.launch_builder(&self.promote_pivots); + lb.arg(&mut m.buf) + .arg(perm) + .arg(&l.buf) + .arg(&r_u) + .arg(&pr_u) + .arg(&fl) + .arg(&tl) + .arg(&st) + .arg(&ls); + unsafe { lb.launch(cfg_1d(trailing_limbs)) }?; + } + let (r_u, pr_u, ls) = (r_piv as u32, pr as u32, l.stride as u32); + let mut lb = stream.launch_builder(&self.zero_pivot_l); + lb.arg(perm).arg(&mut l.buf).arg(&r_u).arg(&pr_u).arg(&ls); + unsafe { lb.launch(cfg_1d(pr)) }?; + } + + // (3) gather U = pivot rows' trailing (pr × trailing_limbs). + let u_buf = unsafe { stream.alloc::(pr * trailing_limbs) }?; + { + let (r_u, fl, pr_u, nc, st) = ( + r_piv as u32, + first_limb as u32, + pr as u32, + trailing_limbs as u32, + stride as u32, + ); + let mut lb = stream.launch_builder(&self.gather_rows); + lb.arg(&u_buf) + .arg(&m.buf) + .arg(perm) + .arg(&r_u) + .arg(&fl) + .arg(&pr_u) + .arg(&nc) + .arg(&st); + unsafe { lb.launch(cfg_1d(pr * trailing_limbs)) }?; + } + + // (4) GEMM C = L(m×pr)·U(pr×t); M[:, first_limb:] ^= C. + let (c_dev, n_padded_lim) = + self.matmul_b1_dev_strided(&l.buf, rows, pr, l.stride, &u_buf, t)?; + self.xor_into_region( + &stream, + &mut m.buf, + &c_dev, + rows, + trailing_limbs, + stride, + first_limb, + n_padded_lim, + )?; + Ok(()) + } + + /// Forward pass of the blocked row reduction over the persistent device + /// buffer (design §4): sweep 64-bit panels left to right, and for each — + /// factor it ([`panel_factor`](Self::panel_factor)), promote the pivot rows' + /// deferred trailing, drop the pivots from the multiplier matrix, and apply + /// the trailing update `M[:, c+b:] ^= L·U` as one wgmma GEMM. Leaves `m` in + /// **row-echelon** form addressed through `perm`: the `rank` pivot rows at + /// perm positions `[0, rank)`, each reduced by earlier pivots; the rest zero. + /// + /// Returns `(perm, rank, pivot_cols)`. Back-substitution to RREF is a + /// separate pass (the mirror-image blocked update over the pivot rows). + pub fn forward_reduce( + &self, + m: &mut DeviceMatrix, + ) -> Result<(CudaSlice, usize, Vec), Box> { + let stream = self.ctx.default_stream(); + let (rows, stride) = (m.rows, m.stride); + let mut perm = self.identity_perm(rows)?; + let mut r = 0usize; + let mut pivot_cols = Vec::new(); + // Panel width in limbs (b = 64·bl columns). Wider panels raise the + // trailing GEMM's contraction dimension pr toward b, reclaiming the ~16× + // K-padding waste. Override with FP_CUDA_BL; otherwise adaptive_bl picks + // the measured optimum (flat at bl≈12–16). + let bl = if let Some(v) = std::env::var("FP_CUDA_BL") + .ok() + .and_then(|v| v.parse::().ok()) + { + v.clamp(1, stride.max(1)) + } else { + adaptive_bl(stride).clamp(1, stride.max(1)) + }; + + // Active-row compaction: park permanently-dead below rows past m_active + // so panel_factor stops scanning them. Re-scanned every COMPACT_PERIOD + // panels to amortize the mark-and-partition cost. + const COMPACT_PERIOD: usize = 4; + let mut m_active = rows; + + // Cooperative multi-CTA promotion (right-looking) replaces the single-CTA + // triangular replay when the matrix is wide enough to amortize the grid + // barriers; otherwise the grid-strided promote_pivots kernel is used. + let use_promote_coop = stride >= 1024; + let (pc_ctas, mut pc_barrier, pc_cond) = if use_promote_coop { + let sms = self + .ctx + .attribute(sys::CUdevice_attribute_enum::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)? + as u32; + let occ = self + .promote_coop + .occupancy_max_active_blocks_per_multiprocessor(256, 0, None)? + .max(1); + // promote is a forward-substitution TRSM (1 grid_sync/pivot), partly + // barrier-bound: a smaller grid gives cheaper barriers while still + // covering the per-pivot trailing XOR — measured +6% at 2^16, +5% at + // 2^17. Cap at 384 (broad H200 optimum); overridable. + let full = (occ * sms).max(1); + let capped = std::env::var("FP_CUDA_PROM_CTAS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(384) + .clamp(1, full); + (capped, stream.alloc_zeros::(1)?, unsafe { + stream.alloc::(bl * 64) + }?) + } else { + (0, stream.alloc_zeros::(1)?, unsafe { + stream.alloc::(1) + }?) + }; + + let mut ppanel = 0usize; + let mut panel_idx = 0usize; + while ppanel < stride { + let bl_eff = bl.min(stride - ppanel); + if panel_idx.is_multiple_of(COMPACT_PERIOD) { + m_active = self.compact_perm(m, &mut perm, r, m_active, ppanel)?; + // No live below rows left ⇒ no column past here can have a pivot; + // the remaining panels would all be empty. Stop the forward sweep + // (huge win for low-rank / rank-deficient inputs). + if m_active <= r { + break; + } + } + // Single wide panel: elementwise cooperative factor + far GEMM. + let mut l = DeviceMatrix { + buf: stream.alloc_zeros::(rows * bl_eff)?, + rows, + cols: bl_eff * 64, + stride: bl_eff, + }; + let (pr, pivcols) = + self.panel_factor_coop(m, &mut perm, &mut l, ppanel, bl_eff, r, m_active)?; + if pr > 0 { + for &q in &pivcols { + pivot_cols.push(q as usize); + } + r += pr; + self.trailing_update( + m, + &perm, + &mut l, + r - pr, + pr, + ppanel + bl_eff, + stride, + use_promote_coop, + &mut pc_barrier, + &pc_cond, + pc_ctas, + )?; + } + ppanel += bl_eff; + panel_idx += 1; + } + stream.synchronize()?; + Ok((perm, r, pivot_cols)) + } + + /// Clear a pivot block's columns from a set of rows *above* it, as one + /// `X·U` GEMM (the back-substitution Schur update, design §4.6). Clears the + /// pivot columns of block `[block_s, block_e)` from the rows at perm positions + /// `[above_start, above_start+above_count)`: gather `X` = those rows' bits at + /// the block's pivot columns, `U` = the (already-RREF) block rows over their + /// trailing, `G = X·U`, then scatter-XOR `G` into the above rows. + /// + /// Unlike the forward trailing update this needs **no promote**: the source + /// (block) rows are already fully reduced and disjoint from the target (above) + /// rows, so it is a clean BLAS3 triangular-solve step. Shared by the outer + /// back-sub loop (`above = [0, s)`) and the recursive within-block TRSM + /// (`above` = the left sub-block). + #[allow(clippy::too_many_arguments)] + fn bs_clear_above( + &self, + m: &mut DeviceMatrix, + perm: &CudaSlice, + piv_dev: &CudaSlice, + pivot_cols: &[usize], + block_s: usize, + block_e: usize, + above_start: usize, + above_count: usize, + ) -> Result<(), Box> { + if above_count == 0 || block_e <= block_s { + return Ok(()); + } + let stream = self.ctx.default_stream(); + let (stride, n) = (m.stride, m.cols); + let bp_eff = block_e - block_s; + let start_limb = pivot_cols[block_s] / 64; + let trailing_limbs = stride - start_limb; + let width_cols = n - start_limb * 64; + let x_stride = bp_eff.div_ceil(64); + // View of the target rows so gather_cols/xor_into_perm read perm at the + // right offset (they index perm[jpos] for jpos < above_count). + let perm_above = perm.slice(above_start..above_start + above_count); + + // X = above rows gathered at the block's pivot columns (above_count × bp_eff). + let x_buf = unsafe { stream.alloc::(above_count * x_stride) }?; + { + let (cs, s_u, cnt, st, xs) = ( + block_s as u32, + above_count as u32, + bp_eff as u32, + stride as u32, + x_stride as u32, + ); + let mut lb = stream.launch_builder(&self.gather_cols); + lb.arg(&x_buf) + .arg(&m.buf) + .arg(&perm_above) + .arg(piv_dev) + .arg(&cs) + .arg(&s_u) + .arg(&cnt) + .arg(&st) + .arg(&xs); + unsafe { lb.launch(cfg_1d(above_count * x_stride)) }?; + } + + // U = the (now RREF) block rows, limbs [start_limb, stride). + let u_buf = unsafe { stream.alloc::(bp_eff * trailing_limbs) }?; + { + let (s_u, fl, pr_u, nc, st) = ( + block_s as u32, + start_limb as u32, + bp_eff as u32, + trailing_limbs as u32, + stride as u32, + ); + let mut lb = stream.launch_builder(&self.gather_rows); + lb.arg(&u_buf) + .arg(&m.buf) + .arg(perm) + .arg(&s_u) + .arg(&fl) + .arg(&pr_u) + .arg(&nc) + .arg(&st); + unsafe { lb.launch(cfg_1d(bp_eff * trailing_limbs)) }?; + } + + // G = X · U (above_count × width_cols); scatter-XOR into the above rows. + let (c_dev, n_padded_lim) = + self.matmul_b1_dev(&x_buf, above_count, bp_eff, &u_buf, width_cols)?; + { + let (s_u, w, st, fl, cs) = ( + above_count as u32, + trailing_limbs as u32, + stride as u32, + start_limb as u32, + n_padded_lim as u32, + ); + let mut lb = stream.launch_builder(&self.xor_into_perm); + lb.arg(&mut m.buf) + .arg(&c_dev) + .arg(&perm_above) + .arg(&s_u) + .arg(&w) + .arg(&st) + .arg(&fl) + .arg(&cs); + unsafe { lb.launch(cfg_1d(above_count * trailing_limbs)) }?; + } + Ok(()) + } + + /// Reduce a pivot block `[s, e)` to RREF **among itself**, elementwise: the + /// triangular solve done as `(e-s)` sequential per-pivot column clears (the + /// `block_reduce_coop` / `block_reduce_rref` kernels). Cheap for a narrow + /// block; the base case of [`block_reduce_rec`](Self::block_reduce_rec). + #[allow(clippy::too_many_arguments)] + fn block_reduce_elem( + &self, + m: &mut DeviceMatrix, + perm: &CudaSlice, + piv_dev: &CudaSlice, + s: usize, + e: usize, + use_coop: bool, + br_barrier: &mut CudaSlice, + br_cond: &CudaSlice, + br_ctas: u32, + ) -> Result<(), Box> { + if e <= s { + return Ok(()); + } + let stream = self.ctx.default_stream(); + let stride = m.stride; + { + if use_coop { + stream.memset_zeros(br_barrier)?; + let (s_u, e_u, st, tc) = (s as u32, e as u32, stride as u32, br_ctas); + let cfg = LaunchConfig { + grid_dim: (br_ctas, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + }; + let mut lb = stream.launch_builder(&self.block_reduce_coop); + lb.arg(&mut m.buf) + .arg(perm) + .arg(piv_dev) + .arg(&s_u) + .arg(&e_u) + .arg(&st) + .arg(br_barrier) + .arg(br_cond) + .arg(&tc); + unsafe { lb.launch_cooperative(cfg) }?; + } else { + let (s_u, e_u, st) = (s as u32, e as u32, stride as u32); + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + }; + let mut lb = stream.launch_builder(&self.block_reduce_rref); + lb.arg(&mut m.buf) + .arg(perm) + .arg(piv_dev) + .arg(&s_u) + .arg(&e_u) + .arg(&st); + unsafe { lb.launch(cfg) }?; + } + } + Ok(()) + } + + /// Recursive **blocked TRSM** reduction of a pivot block `[s, e)` to RREF + /// among itself: split at the midpoint, recurse on the right half, clear its + /// pivots from the left half with one large `X·U` GEMM + /// ([`bs_clear_above`](Self::bs_clear_above)), then recurse on the left half. + /// Below `base_bp` the elementwise + /// [`block_reduce_elem`](Self::block_reduce_elem) runs. + /// + /// This is the BLAS3 form of back-substitution's within-block reduce (design + /// §4.6): it moves the O(bp²·width) triangular work off the elementwise + /// per-pivot clears and onto the tensor cores. Because the source and target + /// rows are disjoint and already reduced, there is **no promote** — the + /// overhead that made the forward-pass recursion a net loss is absent here. + #[allow(clippy::too_many_arguments)] + fn block_reduce_rec( + &self, + m: &mut DeviceMatrix, + perm: &CudaSlice, + piv_dev: &CudaSlice, + pivot_cols: &[usize], + s: usize, + e: usize, + base_bp: usize, + use_coop: bool, + br_barrier: &mut CudaSlice, + br_cond: &CudaSlice, + br_ctas: u32, + ) -> Result<(), Box> { + if e - s <= base_bp { + return self.block_reduce_elem( + m, perm, piv_dev, s, e, use_coop, br_barrier, br_cond, br_ctas, + ); + } + let mid = s + (e - s) / 2; + // Right half to RREF, then clear its pivots from the left half (K = e-mid). + self.block_reduce_rec( + m, perm, piv_dev, pivot_cols, mid, e, base_bp, use_coop, br_barrier, br_cond, br_ctas, + )?; + self.bs_clear_above(m, perm, piv_dev, pivot_cols, mid, e, s, mid - s)?; + // Left half to RREF (its rows now carry no right-half pivot bits). + self.block_reduce_rec( + m, perm, piv_dev, pivot_cols, s, mid, base_bp, use_coop, br_barrier, br_cond, br_ctas, + )?; + Ok(()) + } + + /// Back-substitution: turn the row-echelon form left by + /// [`forward_reduce`](Self::forward_reduce) into full RREF (design §4.6), + /// blocked right-to-left over pivot blocks. For each block of ≤ 64 pivots + /// (perm positions `[s, e)`): reduce it among itself, then clear its pivot + /// columns from every row above `[0, s)` with one `X·U` GEMM. In place over + /// the persistent buffer through `perm`; `pivot_cols` is the ascending pivot + /// column list from the forward pass. + pub fn back_substitute( + &self, + m: &mut DeviceMatrix, + perm: &CudaSlice, + r: usize, + pivot_cols: &[usize], + ) -> Result<(), Box> { + if r == 0 { + return Ok(()); + } + let stream = self.ctx.default_stream(); + let stride = m.stride; + let piv_dev = + stream.clone_htod(&pivot_cols.iter().map(|&q| q as u32).collect::>())?; + + // Cooperative multi-CTA block reduction: spreads each block's per-pivot + // clear across the whole grid. The per-block cooperative launch + grid + // barriers only pay once the block work (≈ bp·stride) is large, so gate on + // a wide matrix; below that the single-CTA kernel wins. Measured (H200): + // neutral at n=2¹⁵ (stride 512), +6% at 2¹⁶, +18% at 2¹⁷. + let use_coop = stride >= 1024; + + // Pivots per back-substitution block. Wider blocks raise the X·U GEMM's + // contraction dimension bp toward TILE_K, cutting its K-padding waste + // (K=64 pads 16×). block_reduce_coop's cost is ~bp-independent (its + // compute and barrier counts both scale with r, not bp), so on the coop + // path we widen bp for free; the single-CTA fallback keeps bp=64 (its + // shared cond[] is sized 64). Override with FP_CUDA_BP. + let bp = if use_coop { + // K=1024 makes the X·U GEMM's contraction an exact TILE_K multiple — + // zero K-padding — and block_reduce_coop is bp-independent. + std::env::var("FP_CUDA_BP") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1024) + } else { + 64 + } + .clamp(1, r); + + const BR_THREADS: u32 = 256; + let sms = self + .ctx + .attribute(sys::CUdevice_attribute_enum::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)? + as u32; + let br_occ = self + .block_reduce_coop + .occupancy_max_active_blocks_per_multiprocessor(BR_THREADS, 0, None)? + .max(1); + // Blocked-TRSM within-block reduce (coop path): recurse each block to + // narrow base blocks + X·U GEMMs. + let use_trsm = use_coop; + // Base ≤ 64: the single-CTA block_reduce_rref's shared cond[] is sized 64. + let base_bp: usize = std::env::var("FP_CUDA_BS_BASE") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(64) + .clamp(1, 64); + + // The block reduce is grid-barrier-bound (2 grid syncs/pivot), and a + // grid_sync's cost scales with the CTA count. With TRSM the reduces are + // narrow base blocks whose per-pivot XOR needs little width, so a small + // grid gives far cheaper barriers — measured bs.block_reduce 334→115 ms + // (2.9×) at 2^16, +12% end-to-end. Cap to 128 CTAs there (the plateau of + // the sweep). Without TRSM the reduce is the full bp=1024 block, whose XOR + // genuinely needs the whole grid, so keep full occupancy. FP_CUDA_BR_CTAS + // overrides. + let br_cap = if use_trsm { 128 } else { br_occ * sms }; + let br_ctas = std::env::var("FP_CUDA_BR_CTAS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(br_cap) + .clamp(1, br_occ * sms); + let mut br_barrier = stream.alloc_zeros::(1)?; + let br_cond = unsafe { stream.alloc::(bp) }?; + + let mut e = r; + while e > 0 { + let s = e - e.min(bp); + + // (1) reduce the block [s, e) to RREF among itself. + if use_trsm { + self.block_reduce_rec( + m, + perm, + &piv_dev, + pivot_cols, + s, + e, + base_bp, + use_coop, + &mut br_barrier, + &br_cond, + br_ctas, + )?; + } else { + self.block_reduce_elem( + m, + perm, + &piv_dev, + s, + e, + use_coop, + &mut br_barrier, + &br_cond, + br_ctas, + )?; + } + + // (2) clear the block's pivot columns from all rows above [0, s). + self.bs_clear_above(m, perm, &piv_dev, pivot_cols, s, e, 0, s)?; + + e = s; + } + stream.synchronize()?; + Ok(()) + } + + /// Full device-resident F₂ row reduction to RREF over one persistent buffer: + /// [`forward_reduce`](Self::forward_reduce) then + /// [`back_substitute`](Self::back_substitute). Mutates `m` to the reduced row + /// echelon form (addressed through the returned `perm`) and returns + /// `(perm, rank, pivot_cols)`. Bit-for-bit equal to + /// `fp::Matrix::row_reduce_blas3` / `row_reduce` (validated in the examples). + pub fn row_reduce_dev( + &self, + m: &mut DeviceMatrix, + ) -> Result<(CudaSlice, usize, Vec), Box> { + let (perm, r, pivot_cols) = self.forward_reduce(m)?; + self.back_substitute(m, &perm, r, &pivot_cols)?; + Ok((perm, r, pivot_cols)) + } +} + +/// Encode a 2D row-major TMA tensor map of UINT32 elements. +fn encode_tma( + dev_ptr: sys::CUdeviceptr, + gdim: [u64; 2], + boxdim: [u32; 2], + row_stride_bytes: u64, + swizzle: sys::CUtensorMapSwizzle_enum, +) -> Result> { + let gstride = [row_stride_bytes]; + let elemstride = [1u32, 1u32]; + let mut tmap = MaybeUninit::::uninit(); + unsafe { + sys::cuTensorMapEncodeTiled( + tmap.as_mut_ptr(), + sys::CUtensorMapDataType_enum::CU_TENSOR_MAP_DATA_TYPE_UINT32, + 2, + dev_ptr as *mut c_void, + gdim.as_ptr(), + gstride.as_ptr(), + boxdim.as_ptr(), + elemstride.as_ptr(), + sys::CUtensorMapInterleave_enum::CU_TENSOR_MAP_INTERLEAVE_NONE, + swizzle, + sys::CUtensorMapL2promotion_enum::CU_TENSOR_MAP_L2_PROMOTION_NONE, + sys::CUtensorMapFloatOOBfill_enum::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE, + ) + .result()?; + Ok(tmap.assume_init()) + } +} + +/// Gather A into plain row-major K-major tiles for TMA 128B swizzle. +/// +/// Output: contiguous tiles, each TILE_M rows × KL u64s (64 × 128 bytes). The +/// TMA applies the 128B swizzle on load, so the host layout is the natural +/// row-major sub-block: tile row `row` holds K bits `kk*TILE_K .. +TILE_K` of +/// global row `bi*TILE_M + row`, zero-padded out of bounds. +/// +/// Tiles are ordered: for K-chunk kk=0..k_chunks-1, then M-tile bi=0..m_tiles-1. +fn interleave_a(a: &[u64], m: usize, k: usize) -> Vec { + let sa = k / 64; + let k_chunks = k / TILE_K; + let m_tiles = m / TILE_M; + let tile_u64s = TILE_M * KL; + let mut out = vec![0u64; k_chunks * m_tiles * tile_u64s]; + + for kk in 0..k_chunks { + for bi in 0..m_tiles { + let base = (kk * m_tiles + bi) * tile_u64s; + for row in 0..TILE_M { + for kl in 0..KL { + let global_row = bi * TILE_M + row; + let global_kl = kk * KL + kl; + let val = if global_row < m && global_kl < sa { + a[global_row * sa + global_kl] + } else { + 0 + }; + out[base + row * KL + kl] = val; + } + } + } + } + out +} + +/// Pre-transpose B into plain row-major K-major tiles for TMA 128B swizzle. +/// +/// Each (k_chunk, column group) tile is NB = NG*64 rows (= the NG*64 output +/// columns of the group) × KL u64s (= TILE_K K bits); the consumer feeds it to +/// MSTRIPS m64n128 wgmmas that share it. Operand row `lg*64 + jj` is output +/// column `cg*NG*64 + lg*64 + jj`; element `[..][kl] bit` is bit `jj` of +/// `B[k_chunk*TILE_K + kl*64 + bit][cg*NG + lg]`. +/// Groups whose limb runs past `n_lim` are left zero-padded. Output is +/// row-major; the TMA applies the swizzle on load. +fn transpose_b(b: &[u64], k: usize, n_lim: usize) -> Vec { + let k_chunks = k / TILE_K; + let ng = NG as usize; + let n_groups = n_lim.div_ceil(ng); + let tile = ng * 64 * KL; // 256 rows × KL u64 + let mut out = vec![0u64; k_chunks * n_groups * tile]; + let mut buf = [0u64; TILE_K]; + + for kk in 0..k_chunks { + for cg in 0..n_groups { + let base = (kk * n_groups + cg) * tile; + for lg in 0..ng { + let limb = cg * ng + lg; + if limb >= n_lim { + continue; // padded column group → leave zeros + } + for (i, slot) in buf.iter_mut().enumerate() { + let br = kk * TILE_K + i; + *slot = if br < k { b[br * n_lim + limb] } else { 0 }; + } + for jj in 0..64usize { + let j = lg * 64 + jj; // operand row within the 256-col tile + for kl in 0..KL { + let mut val: u64 = 0; + for bit in 0..64usize { + val |= ((buf[kl * 64 + bit] >> jj) & 1) << bit; + } + out[base + j * KL + kl] = val; + } + } + } + } + } + out +} + +fn pad_2d(src: &[u64], rows: usize, stride: usize, nr: usize, ns: usize) -> Vec { + if rows == nr && stride == ns { + return src.to_vec(); + } + let mut out = vec![0u64; nr * ns]; + for r in 0..rows { + let n = stride.min(ns); + out[r * ns..r * ns + n].copy_from_slice(&src[r * stride..r * stride + n]); + } + out +} diff --git a/ext/crates/fp/BLAS3-GPU-HANDOFF.md b/ext/crates/fp/BLAS3-GPU-HANDOFF.md new file mode 100644 index 0000000000..4d3375da84 --- /dev/null +++ b/ext/crates/fp/BLAS3-GPU-HANDOFF.md @@ -0,0 +1,277 @@ +# BLAS3 F₂ row reduction — GPU handoff (for an H200-enabled agent) + +> **STATUS (device port done, H200-validated).** Phases 3–5 and 7 are +> implemented and validated bit-exact on an H200 NVL (sm_90): +> - **Phase 3** — `DeviceMatrix` + persistent-buffer glue; `matmul_b1_dev` +> (device-in/device-out GEMM, on-device packing) + `xor_into_region`/ +> `gemm_xor_into` fused epilogue. Gates: `matmul_b1_dev_demo`, +> `gemm_xor_into_demo`. +> - **Phase 4** — `panel_factor` base kernel (b=64, virtual `perm`). Gate: +> `panel_factor_demo` (vs a CPU transliteration; full- and low-rank panels). +> - **Phase 5** — `forward_reduce` (forward echelon) + `back_substitute` → full +> RREF `row_reduce_dev`, all device-resident. Gates: `forward_reduce_demo`, +> `row_reduce_demo` (`row_reduce_dev == fp::Matrix::row_reduce`, bit-exact). +> - **Phase 7** — `fp::Matrix::row_reduce` dispatches to the GPU above +> `FP_CUDA_THRESHOLD`. Test: `tests/cuda_dispatch::gpu_row_reduce_matches_cpu`. +> +> **Phase 6 — optimization passes (done, H200-validated).** Profiling +> (`FP_CUDA_PROF` per-phase timers + `probe_pr` flat-K microbench) showed the +> single-CTA `panel_factor` — not the trailing GEMM — dominated the forward pass +> (~76% of GPU time). The passes, each kept bit-exact against the CPU oracle: +> - **Multi-CTA `panel_factor_coop`** — grid-parallel find-first + masked-XOR with +> a self-contained atomic grid barrier between the 64 bit-steps, cooperative +> launch (no cudadevrt). 76% → ~21%. + - **Wide forward panels** — `panel_factor_coop` factors a bl-limb panel natively +> (inline intra-panel Schur update across all bl limbs), so one wide trailing +> GEMM (K=pr≤b) reclaims the K-padding waste. Width is adaptive +> (`adaptive_bl(stride)` = stride/128 on the coop-promote path, stride/256 below; +> env `FP_CUDA_BL`). +> - **Cooperative multi-CTA back-substitution kernels** — `block_reduce_coop` (each +> block's clear spread over the grid) and, since its cost is ~block-width- +> independent, a **wide back-sub block** `bp = 512` (K=64 → 512, cutting the X·U +> GEMM's 16× K-padding; env `FP_CUDA_BP`). +> - **Cooperative right-looking `promote_coop`** — the O(pr²) triangular replay, +> reformulated so each pivot's contribution to later pivots is one grid-parallel +> XOR (grid barrier between pivots) instead of a few-CTA per-column serial loop. +> - **Multi-CTA `promote_pivots`** (parallel over columns) and a low-barrier +> `block_reduce_rref` (~bp² → ~2·bp `__syncthreads`) — the single-CTA fallbacks +> used below the coop threshold (stride < 1024). `FP_CUDA_NO_COOP` forces them. +> - **Skip zeroing** fully-overwritten GEMM/gather buffers (uninitialized alloc). +> - **Active-row compaction** (§8.2) + rank-exhaustion early-exit (`mark_live` / +> `compact_perm`; env `FP_CUDA_NO_COMPACT`). +> - **Separate, higher row-reduce dispatch threshold** (`FP_CUDA_RR_THRESHOLD`, +> default 8192 — the measured H200 crossover vs multi-threaded M4RI). +> +> Results (half-rank square, bit-exact vs CPU `row_reduce`; device incl. up/download +> excluded), vs the pre-Phase-6 device baseline and multi-threaded CPU: +> +> | n | device before | device after | vs M4RI | vs CPU BLAS3 | Tbop/s | +> |------|------|------|------|------|------| +> | 32768 | 2.69 s | 0.59 s | 18.0× | 18.0× | 60 | +> | 65536 | 11.9 s | 1.79 s | — | 36.6× | 157 | +> | 131072 | 74.9 s | 5.93 s | — | — | 380 | +> +> Speedup grows with size (≈4.5–12.6× device-over-device, larger vs CPU). Full-rank +> 32768² is 21× vs M4RI. The profile is again balanced (panel/gemm/promote/ +> block_reduce/bs-gemm ≈ 33/19/18/17/9 % at n=2¹⁷). **Still open (need major +> restructures for the last balanced slices):** an intra-panel *GEMM* (tensor cores +> for the inline panel XOR, now the top cost at wide bl) and a blocked TRSM for +> promotion; plus the smaller pipeline/pack/fused-epilogue passes. All build with +> CUDA 12.4 nvcc; `cargo run -p fp-cuda --example ` for each gate. + +--- + +> **TL;DR.** The CPU algorithm is done, proptested bit-exact against +> `Matrix::row_reduce`, and benchmarked: on 4 CPU cores its cost crosses M4RI +> around `n ≈ 1.2·10⁵` (the real workload). Your job is the **device-resident +> port** — run the whole reduction as a sequence of kernels over *one* persistent +> GPU buffer, so PCIe transfer is paid twice (upload + download), not once per +> panel. The F₂ GEMM kernel you need already exists and is **H200-validated**; +> what's missing is a device-resident API around it and one new panel kernel. +> +> Read `BLAS3-ROW-REDUCTION.md` first — this file is the execution wrapper, that +> file is the design (§ references below point into it). + +--- + +## 0. What's already true (don't redo) + +- **CPU algorithm** (`src/matrix/blas3.rs`, `Matrix::row_reduce_blas3`): blocked + forward-echelon + blocked back-substitution, limb-wise panel. Proptested + bit-for-bit against `row_reduce` (RREF **and** pivots) incl. rank-deficient + generators (`src/matrix/blas3.rs` tests + `proptest-regressions/`). **This is + your executable spec and your oracle.** +- **F₂ GEMM kernel** (`fp_cuda_hopper` branch, `crates/fp-cuda`): Hopper + `wgmma.b1` matmul, **validated on H200 NVL** (Phases 3–12), ~thousands of + binary TOPS kernel-only. But its only entry point, `matmul_b1_raw`, does + `clone_htod(A) + clone_htod(B) + alloc(C) + clone_dtoh(C)` **per call** — the + crate's own README notes end-to-end throughput is "dominated by host + marshalling." That per-call marshalling is exactly what you must NOT loop over + (design §1.1). +- **Bench harness**: `examples/reduce_scaling.rs` (one-shot large sizes), + `benches/reduce_blas3.rs` (criterion). Reuse for GPU numbers. + +Current CPU trend (4 threads, half-rank, AVX-512 host), for context on what +"good" looks like — the GPU should blow past M4RI here: + +| n | M4RI | `row_reduce_blas3` | ratio | +|---|---|---|---| +| 16384 | ~2 s | 4.0 s | 1.8× | +| 32768 | ~26–31 s | 45.9 s | 1.46× | + +--- + +## 1. Repo setup (combine the two branches) + +You need the CPU algorithm **and** the GEMM kernel in one tree: + +- CPU algorithm: branch `claude/blas3-row-reduction-lgjz3y` (off `master`). +- GEMM kernel: branch `fp_cuda_hopper` (off `master`). + +They're disjoint except `ext/crates/fp/Cargo.toml` (one adds a `[[bench]]`, the +other adds `fp-cuda` deps + a `cuda` feature — a trivial both-sides merge). Merge +`fp_cuda_hopper` into `claude/blas3-row-reduction-lgjz3y` (or a fresh integration +branch), resolve the `Cargo.toml` union, and you have `row_reduce_blas3` next to +`fp_cuda::matmul_b1_raw`. + +Sanity gates before any new work (H200 box): + +```sh +cargo build -p fp-cuda # PTX JITs on the device +cargo run -p fp-cuda --example matmul_b1_demo # CPU↔GPU bit-exact GEMM sweep +cargo test -p fp --features proptest blas3 # CPU oracle still green +``` + +If the GEMM demo is bit-exact, the kernel is sound on your H200 and every number +below has a trustworthy oracle. + +--- + +## 2. The one rule + +**One device allocation for the whole reduction.** Upload the matrix once, run +every step as an in-place kernel over that buffer, download once. Host↔device +traffic must stay ≈ 2× the matrix regardless of panel count. If you find yourself +calling `matmul_b1_raw` (which round-trips the host) inside the panel loop, stop +— that re-marshals overlapping slabs `n/b` times and PCIe becomes the ceiling +(design §1.1). Everything else follows from this. + +Corollary constraints (design §1.2): confine all **column**-indexed work to the +narrow panel; make wide work either a GEMM or a contiguous whole-limb row-XOR; +never permute columns; and make row "swaps" **virtual** via a `perm` index vector +(design §4.3) so the multi-hundred-MB buffer never physically shuffles. (The CPU +code physically swaps rows — fine for the oracle, wrong at GPU scale. This is the +one place the GPU port intentionally diverges from `blas3.rs`.) + +--- + +## 3. Device API to build in `fp-cuda` (design §6) + +Add a handle-based, device-resident layer beside the existing one-shot function: + +```rust +pub struct DeviceMatrix { /* CudaSlice + rows, cols, stride, on-device perm */ } + +impl GpuContext { + fn upload(&self, m: &Matrix) -> DeviceMatrix; // one H2D + fn download(&self, dm: &DeviceMatrix) -> Matrix; // one D2H, applies perm + + // in-place device kernels over the persistent buffer: + fn panel_factor(&self, dm, limb_lo, limb_hi, col_hi, r, /*out*/ l, pivcols) -> pr; // §5 + fn gemm_xor_into(&self, dm, factors, rows, cols, dst_limb); // wgmma.b1 + fused XOR epilogue + fn gather_bits(&self, dm, rows, cols) -> DeviceMatrix; // column-gather for X / back-sub +} +``` + +Two deltas from today's kernel do most of the work: + +1. **Persistent buffers.** Reuse the *compute* core of `cuda_kernels/matmul_b1.cu` + unchanged; change only the host glue so operands are device pointers into the + resident buffer, not fresh `htod` copies. +2. **Fused XOR-accumulate epilogue.** Both trailing update (§4.4) and back-sub + (§4.6) are `M[region] ^= factors · rows`. Have the GEMM store step XOR its + result into an existing device region instead of writing a fresh C and copying + it back — saves an alloc + a D2H per panel and does the update for free. + +Everything the CPU driver does in `row_reduce_blas3` maps onto these: the forward +`L·U` trailing GEMM and the back-sub `X·U` GEMM both become `gemm_xor_into`. + +--- + +## 4. The one genuinely new kernel: `panel_factor` (design §5) + +The panel is `m × (b/64)` limbs — small (a few MB), stays in L2 across the sweep. +It's the only place with column-indexed / sequential work. Port the CPU Step A +loop (`blas3.rs`, the `for q in col_lo..col_hi` block) to a device kernel: + +- **Base case `b = 64` (one limb wide):** each active row contributes one `u64`; + factor by sweeping the 64 bit-positions — for bit `j`, a `find-first` reduction + over `m` rows picks the pivot (mark its `perm` slot), then a **row-parallel + masked XOR** clears bit `j` from every other row and records the multiplier + into `L`. 64 coalesced passes over `m` words. +- **Wider `b` (256–1024):** split into 64-wide sub-panels, factor the leftmost + with the base kernel, update the rest of the *panel* with a small GEMM, recurse + (§5(2)). Start with the base kernel at modest `b`; add this only if the panel + shows up in profiles. +- Emit to host only `pr` + the pivot columns (a few hundred ints/panel). `L`, the + reduced pivot rows, and `perm` stay on device. + +The CPU `blas3.rs` promotion/deferral logic (why forward-only elimination keeps +the pivot trailings stable) is documented at §4.6 — preserve that structure; it's +what makes the deferred GEMM correct. + +--- + +## 5. Order of work + validation gates (design §10, Phases 3–7) + +Each phase has a bit-exact oracle: the GPU result, downloaded, must equal +`row_reduce_blas3` on the same input (which is itself `== row_reduce`). + +| Phase | Deliverable | Gate | +|---|---|---| +| 3 | `DeviceMatrix` + persistent-buffer glue; `gemm_xor_into` fused epilogue | a single trailing-update GEMM matches CPU Step B bit-for-bit | +| 4 | `panel_factor` base kernel (b=64) + virtual `perm` | one panel factorization matches CPU Step A (pivots + `L`) | +| 5 | wire the §4 driver end-to-end on device (forward + blocked back-sub) | full `row_reduce_blas3` GPU == CPU, bit-exact, across the shape grid | +| 6 | active-row compaction (§8.2); wider panel GEMM (§5(2)) if profiled | still bit-exact; measure speedup | +| 7 | dispatch in `fp`: `row_reduce` picks GPU above a size/rank threshold | pick threshold from the crossover (§10.5) | + +Recommended micro-oracles while building: extend `examples/reduce_scaling.rs` to +run the GPU path beside `row_reduce_blas3` and `assert_eq!` the full matrices, at +sizes from 64 up to 2^15+. Bit-exactness at small sizes catches ~every kernel +bug before the big runs. + +--- + +## 6. Risks / gotchas + +- **Marshalling creep** (§2) — the failure mode. Audit that the panel loop issues + zero `htod`/`dtoh` beyond the initial upload and the scalar pivot-column + read-backs. +- **`perm` indirection** — every kernel dereferences `M[perm[i]]`. Get this + uniform early; retrofitting it is painful. +- **Padding is your friend** (design, "WLOG pad to a limb"): columns are stored + padded to a limb boundary and rows to a multiple of 64; treat the padding as + genuine zeros so the last partial limb needs no special case. The CPU code + already relies on this. +- **GEMM operand layout** — `matmul_b1.cu` expects a specific TMA/bit-pack layout + (`af251a5`, the n256 fragment). `gemm_xor_into` must feed `L`/`U`/`X` in that + layout from device memory; reuse `matmul_b1_raw`'s packing logic, just without + the host round-trip. +- **fp-cuda's own `HANDOFF.md`** documents an older Phase 8–9 cluster/multicast + batch; the README says Phases 8–12 are since H200-validated. If a GEMM shape + regresses, that's the kernel's bisect trail, not yours. +- **Rank deficiency (§8)** — the workload is rank ≈ m/2. Work is already ∝ rank + (empty panels cost ~0); add active-row compaction (§8.2) via `perm` partition + so the GEMM's tall dimension shrinks toward R. It's a `perm` reshuffle, no byte + movement. + +--- + +## 7. H200 specifics + +- Compute capability sm_90a — the kernel targets exactly this; already validated + on H200 NVL. No kernel changes needed to *run*. +- 141 GB HBM3e, ~4.8 TB/s. A 100 000² F₂ matrix is ~1.25 GB — fits comfortably + with room for the panel/`L`/`U`/`G` temporaries, so the whole reduction can be + device-resident with no tiling of the *matrix* itself. +- The kernel's known throughput cliff was L2-residency of B beyond 16384 (fp-cuda + README/HANDOFF); the row-reduction GEMMs are tall-skinny × wide (`m × pr` · + `pr × T`), a different shape — profile them specifically, and lean on the + Phase 8–12 rasterization/cluster work already in the kernel. + +--- + +## 8. Definition of done + +1. `row_reduce_blas3` runs entirely on the H200 over one resident buffer, + bit-exact vs the CPU version at every tested size (64 … ≥ 2^15). +2. Host↔device traffic is ≈ 2× the matrix (verify with a profiler / counters), + independent of panel count. +3. On the target regime (n ~ 10⁵, rank ~ n/2) the GPU path is decisively faster + than M4RI *and* than CPU `row_reduce_blas3`; report the numbers next to the + §10.4 table. +4. `row_reduce` dispatches to it above a size/rank threshold (§10.5), M4RI below. + +The CPU side is the finished skeleton and the oracle; you're moving it onto +silicon that's ~1000× more parallel than the 4 cores that already put the +crossover at the real workload size. diff --git a/ext/crates/fp/BLAS3-ROW-REDUCTION.md b/ext/crates/fp/BLAS3-ROW-REDUCTION.md new file mode 100644 index 0000000000..e113d0a475 --- /dev/null +++ b/ext/crates/fp/BLAS3-ROW-REDUCTION.md @@ -0,0 +1,614 @@ +# BLAS3 (GEMM-based) row reduction over F₂ — design plan + +> Status: **CPU path complete** (Phases 1/1b/1c — `src/matrix/blas3.rs`, +> `Matrix::row_reduce_blas3`): proptested bit-exact vs `row_reduce`, benchmarked, +> 4-core crossover with M4RI at `n ≈ 1.2·10⁵` (§10.4–10.5). The device-resident +> GPU phases (3–7) remain to be built on a Hopper/H200 GPU — **see the execution +> handoff in `BLAS3-GPU-HANDOFF.md`**, which wraps §5/§6/§8/§10 into ordered, +> gated tasks for a GPU-enabled agent. +> +> Goal: reduce a large, highly rank-deficient dense matrix over F₂ to reduced +> row echelon form (RREF) — the semantics of `Matrix::row_reduce` — with the +> bulk of the work expressed as F₂ matrix–matrix products (`wgmma.b1`), so it +> rides the `fp-cuda` GEMM kernel instead of the scalar/AVX M4RI path. +> +> Target workload: `m ≈ 100 000` rows, `n` comparable, **rank `R ≈ m/2`**. +> +> **What Phase 1 built.** A CPU-resident blocked reduction that runs against the +> generic `<&Matrix as Mul>::mul` (AVX-512 tiled kernel today, GPU once wired +> in), proptested bit-for-bit against `Matrix::row_reduce` (RREF *and* pivots), +> including rank-deficient generators. It is structured as **forward blocked +> elimination to echelon form** (the GEMM-heavy part, §4) followed by a +> **back-substitution** pass to RREF (§4.6). The Gauss–Jordan-in-one-pass sketch +> the earlier draft of §4 described does *not* work with a deferred trailing +> update: back-substitution keeps mutating the pivot rows' trailing, so they are +> not stable operands for the GEMM. Forward-only elimination only ever reduces a +> pivot by an *earlier* (stable) pivot, which is what makes the deferred GEMM +> correct — see §4.6. + +--- + +## 1. The two constraints that dictate the design + +Everything below is shaped by two facts the naive "call the fast GEMM in a loop" +approach gets wrong. + +### 1.1 The computation must stay **GPU-resident** + +The current GPU entry point is `fp_cuda::matmul_b1_raw`, dispatched per-multiply +from `fp/src/blas/cuda.rs::try_mul`. Reading `matmul_b1_inner` (fp-cuda +`src/lib.rs`), **every call**: + +1. host-repacks A into TMA interleave and transposes+packs B (`bt`), +2. `clone_htod(A)`, `clone_htod(B)` — two H2D copies, +3. `alloc_zeros(C)` + launch, +4. `clone_dtoh(C)` — one D2H copy. + +For an `M×N` product the transfers are `Θ(MK + KN + MN)` bytes. A blocked row +reduction performs `Θ(n/b)` trailing GEMMs. If each is an independent +`matmul_b1_raw`, we pay a **full PCIe round trip per panel**, re-uploading +overlapping regions of the same matrix `n/b` times. At 1.25 GB for the working +matrix and ~tens of GB/s over PCIe, transfer — not compute — sets the ceiling. +The Phase 3–9 kernel peaks at ~5 800 binary TOPS on H100; that throughput is +unreachable if the operands are re-marshalled from the host each panel. + +**Consequence.** The matrix must live in **one persistent device allocation** +for the whole reduction. Panel factorization, the trailing GEMM, and the +in-place update all run as device kernels over that buffer. The host sees only: +the initial H2D of the matrix, a trickle of scalars per panel (pivot columns + +rank, ~kilobytes), and one final D2H. Total bus traffic ≈ **2× the matrix**, +independent of `n/b`. This is a new device-resident API surface in `fp-cuda` +(§6), *not* a caller of the existing `try_mul`. + +### 1.2 Column operations are **expensive**; row operations are cheap + +Layout (`matrix_inner.rs`): row-major, each row a run of 64-bit limbs; column +`j` is bit `j%64` of limb `j/64`. Therefore: + +| Operation | Cost | Why | +|---|---|---| +| XOR one row-slice into another (contiguous limbs) | **cheap** | coalesced; one thread per limb-column streams contiguous memory | +| Whole-matrix GEMM `C = A·B` | **cheap** (relative to work) | the `wgmma.b1` kernel | +| Read/scan/gather a single **column** across rows | **expensive** | strided by `stride` limbs, one useful *bit* per limb loaded, no coalescing; pivot search and column-gather live here | +| Swap two **columns** | **expensive** | strided RMW of single bits | +| Swap two **rows** | cheap-ish, but moves `stride` limbs × … | avoid at scale — see virtual permutation (§4.3) | + +So the design must (a) confine every column-indexed access to the **narrow +panel** (width `b`, a few limbs), never the full width, and (b) express all +wide, dominant work as GEMM or as contiguous row-XOR. It must **never** permute +columns, and should avoid physically moving rows in the 1.25 GB buffer. + +--- + +## 2. Why the existing code can't be reused as-is + +- `Matrix::row_reduce` (`matrix_inner.rs:674`) is **M4RI** (Method of Four + Russians): build a table of `2^k` combinations of `k` pivot rows, reuse it to + reduce the rest. Arithmetic intensity is ~O(1) per bit touched — a BLAS2-class + algorithm. It cannot feed a tensor-core GEMM and it is inherently host/CPU. +- `fp/src/blas` (the AVX-512 tiled GEMM) and `fp-cuda` give us a *fast GEMM*, + but no *elimination*. The gap is precisely a row-reduction whose inner loop is + GEMM. +- `cuda.rs::try_mul` is the wrong granularity (per-multiply marshalling, §1.1). + +The literature calls the object we want the **PLE decomposition** `A = P·L·E` +(P a row permutation, L unit lower-triangular on the pivot rows/cols, E a row +echelon form). It is the rank-revealing generalization of PLU, and the standard +result is that it **reduces to matrix multiplication** — this is exactly the +BLAS3 property we need. References in §8. RREF is a cheap back-substitution away +from PLE (and, in the Gauss–Jordan variant below, produced directly). + +--- + +## 3. Cost model and the rank-deficiency win + +Let `m` rows, `n` cols, rank `R`. Right-looking blocked elimination with panel +width `b`: + +- **Trailing GEMM per panel:** `L (m × pr) · U (pr × (n−c−b))`, where `pr ≤ b` is + the number of pivots found in that panel. Summed over panels, the GEMM inner + dimension totals `R` (every pivot is used exactly once as a GEMM row), and the + trailing width averages `~n/2`. **Total GEMM work `≈ m · n · R / 64` bit-ops** + (the `/64` is limb packing; tensor cores add another large constant factor). + Because the cost carries a factor `R`, not `n`, **a rank-½ matrix is ~2× cheaper + than full rank for free** — the single most important property for this + workload. Empty column panels (no pivots) contribute *zero* GEMM. + +- **Panel factorization per panel:** confined to width `b`. Total + `≈ m · R · b / 64` for the eliminations plus `≈ m · R` bit-reads for pivot + search (the only column-indexed term). With `b ≪ n` this is a lower-order term + vs. the trailing GEMM (ratio `≈ b / n`). Choosing `b` in `[256, 1024]` keeps + it negligible while keeping the GEMM's inner dimension `pr` large enough to + saturate the kernel. + +Net: **`O(m·n·R)` total, GEMM-dominated, ∝ rank.** The recursive PLE variant +(§7) lowers the panel term further and reaches the sub-cubic +`O(m·n·R^{ω−2})` shape, but the iterative scheme already puts essentially all +the flops in the GEMM. + +--- + +## 4. The algorithm: right-looking blocked Gauss–Jordan, GPU-resident + +Produces global RREF directly (clears above *and* below pivots), matching +`row_reduce` semantics (reduced form + pivot list). One device buffer `M` +holds the matrix throughout. + +State: `r` = pivots found so far (also the number of finished pivot rows, kept +at the top via the virtual permutation of §4.3). Panels sweep columns left to +right; `c` is always a multiple of 64 and `b` a multiple of 64, so a panel +occupies whole limbs `[c/64, (c+b)/64)` of each row — critical for §4.4. + +For each panel `[c, c+b)`: + +### 4.1 Step A — panel factorization (the only column-op region) + +Reduce the sub-block `M[active, c:c+b]` to RREF **within the panel columns +only**, discovering up to `b` pivots. Row operations here touch **only the panel +limbs** `[c/64, (c+b)/64)` — the trailing columns are deliberately left stale +and fixed up by the GEMM in Step B. This is where all the (narrow) column work +lives; §5 details the device kernel. Outputs: + +- `pr` new pivots and their absolute pivot columns `q_0 < … < q_{pr−1} ∈ [c,c+b)`; +- the `pr` pivot rows logically placed at positions `[r, r+pr)` (via the + permutation, §4.3), each `1` at its own pivot column and `0` at the other + pivot columns (RREF among pivots); +- the **multiplier matrix** `L`: for every non-pivot row `j`, an `pr`-bit row + recording which pivot rows were added to `j` during the panel sweep. This is + captured *as the sweep runs* (§4.2) and equals the panel-pivot-column bits of + the pristine rows. + +### 4.2 Why the multipliers `L` are exactly the pivot-column bits + +The panel sweep is Gauss–Jordan restricted to panel limbs. When it clears pivot +column `q_k` from row `j`, it adds pivot row `k` iff the current bit `M[j, q_k]` +is 1 — record that bit as `L[j, k]`, then clear it. Because the pivot rows are +kept mutually reduced (each pivot row is 0 at the *other* pivot columns), adding +pivot row `k′≠k` to row `j` never disturbs `M[j, q_k]`. Hence each captured bit +equals the **pristine** `M[j, q_k]`, and the `L[j, ·]` are independent — no +ordering hazard. A row never adds itself, so the pivot rows get `0` on their own +column and are correctly reduced by *later* pivots in the same panel. The trailing +GEMM then applies these same multipliers to the trailing columns. This is the +in-place-`L` trick of blocked LU, specialized to F₂ (the multiplier is a single +bit; "keep `L`" means "read the bit before you clear it"). + +### 4.3 Virtual row permutation (avoid moving 1.25 GB) + +"Move the pivot row up to position `r`" must **not** physically copy rows in the +device buffer. Keep an on-device permutation/index vector `perm` (length `m`). +"Swap rows" swaps two entries of `perm`; kernels dereference `M[perm[i]]`. The +matrix bytes never move during the sweep. A single optional physical compaction +(gather by `perm`) can be done once at the very end when materializing the +result — or never, if the caller reads rows through `perm`. Row *swaps* are thus +`O(1)` index writes, not `O(stride)` limb moves. + +### 4.4 Step B — trailing update (the GEMM), in place on device + +Apply the recorded eliminations to the trailing columns in one GF(2) product: + +``` +U = pivot rows [r, r+pr) restricted to columns [c+b, n) # pr × T, T = n−c−b +G = L · U # m × T (wgmma.b1) +M[:, c+b:n] ^= G # in-place XOR +``` + +`L` is `m × pr` (pr ≤ b, narrow). The product `G` is `m × T`. Because `c+b` is a +multiple of 64, the trailing region begins at a limb boundary: `U` is a +contiguous limb-slice of the pivot rows, and the update is a **whole-limb XOR** +of `G` into `M`'s trailing limbs — `G.stride == (#trailing limbs of M)` exactly, +so it is a coalesced row-parallel XOR, no column indexing. Pivot rows get `L`-row += 0 on their own future and are left untouched by the XOR (their panel columns +were already finished in Step A), so they stay correctly reduced. + +Then `r += pr` and advance to the next panel. After the last panel, `M` +(read through `perm`) is in global RREF; the pivot list is `q_*` gathered across +panels. + +### 4.5 Correctness argument (for validation against `row_reduce`) + +Invariant after processing columns `[0, c+b)`: `M` is in global RREF with respect +to the pivots found so far, over columns `[0, c+b)`, and rows `[0, r)` are those +pivots. Step A extends the reduced region across the panel columns and records +`L`; Step B extends it across the trailing columns with the identical multipliers +(the split is only *where* the same row-combinations are applied — panel limbs by +XOR in A, trailing limbs by GEMM in B). No double application: A touches only +panel limbs, B only trailing limbs. Therefore the invariant holds inductively and +the final matrix is the RREF. This is checkable exhaustively on CPU (§9) with no +GPU. + +### 4.6 What Phase 1 actually implements: forward echelon + back-substitution + +§4.1–4.5 describe a one-pass Gauss–Jordan (clear above *and* below per panel). +That does **not** compose with a deferred trailing GEMM, and the reason is worth +recording. If a panel clears its pivot columns from rows *above* (earlier +pivots), it mutates those pivot rows' trailing. But a later panel's pivot may be +a row that an earlier panel reduced — and its deferred trailing has to be +"realized" from the earlier pivots' trailing at GEMM time. If those trailings +keep changing (because of ongoing back-substitution), the realized value is +wrong. The 3×65 counterexample that surfaced this: three pivots that reduce each +other, and only the trailing (limb 1) column comes out wrong. + +The fix — standard for blocked LU/PLE — is to split the two directions: + +- **Forward pass (blocked, the GEMM part).** Each panel does forward elimination + only: clear pivot columns from rows *below* the pivots, deferring their + trailing to the `L·U` GEMM of §4.4. A new pivot row is "promoted" by replaying + the earlier *this-panel* pivots into its trailing — and those earlier pivots + are **stable**, because forward elimination never reduces a pivot by a *later* + one. This yields row-echelon form: pivot rows at `[0, R)`, each reduced by + earlier pivots; rows `[R, m)` zero. This is where the `O(m·n·R)` GEMM work is. +- **Back-substitution pass → RREF.** Clear each pivot column from the rows above + it, processing pivots high-to-low so every source row is already fully reduced + before it is used (adding it back can't reintroduce a later pivot column). + +Phase 1 does the back-substitution with plain full-width row operations — +`O(R²·n)`, correct and simple, but **not yet BLAS3**. Because it runs on only the +`R` pivot rows and has the identical `L·U` structure as the forward update +(mirror image, right-to-left over the pivot rows), it blocks into GEMM the same +way; that is the first Phase-2 task (and for `R ≈ m/2` it is ~half the total +work, so it must be blocked before the GPU port pays off). The forward pass — the +novel, dominant, GEMM-based part — is what Phase 1 validates. + +--- + +## 5. The panel-factorization kernel (design of the hard part) + +The panel `P = M[active, c:c+b]` is `m × (b/64)` limbs — e.g. `m=1e5`, `b=256` +→ `1e5 × 4` u64 = 3.2 MB, a small working set that stays in device memory (and +largely in L2) across the sweep. Two implementation strata: + +1. **`b = 64` (one limb wide) base kernel.** Each active row contributes one + u64. Factoring is "reduce `m` u64 words to echelon over GF(2)": iterate the 64 + bit-positions; for bit `j`, find a not-yet-pivot word with bit `j` set (a + `find-first` reduction over `m` bits — the lone column op), mark its `perm` + slot as pivot `k`, then in a **row-parallel masked XOR** add that pivot word + into every other word whose bit `j` is set, capturing the cleared bit into + column `k` of `L`. 64 bit-steps, each a coalesced pass over `m` words. + +2. **`b = 256…1024` via the base kernel + intra-panel GEMM (recursive/blocked + within the panel).** Split the panel into 64-wide sub-panels; factor the + leftmost with (1), then update the rest of the *panel* with a small GEMM + (same L·U shape, but width `≤ b`), recursing. This is the panel-level image of + §7 and keeps even the panel's dominant work in GEMM. Start with (1) at modest + `b`; add (2) only if the panel term shows up in profiles. + +Design points: +- **No column swaps ever.** Pivot selection permutes *rows* (via `perm`), and + free (non-pivot) columns are simply skipped and recorded — the pivot column + list `q_*` carries the column identity; columns are never physically moved. +- **`find-first` down a 64-bit column** is the only irreducible column read. + It is `m` bit-reads per column, `b` columns per panel, `n/b` panels → `m·R` + total (§3) — lower order, and itself a coalesced reduction (load each active + row's panel limb once; test the bit). +- The kernel emits to host only `pr` and `q_0..q_{pr−1}` (a few hundred ints per + panel). Everything else (`L`, the reduced pivot rows, `perm`) stays on device. + +--- + +## 6. Required `fp-cuda` device-resident API (new surface) + +The current crate exposes one shot `matmul_b1_raw(ctx, a, m, k, b, n) -> Vec` +(host in, host out). Row reduction needs a **handle-based device API** so buffers +persist across kernels. Sketch: + +```rust +pub struct DeviceMatrix { ptr: CudaSlice, rows, cols, stride, /* on device */ } + +impl GpuContext { + fn upload(&self, m: &Matrix) -> DeviceMatrix; // one H2D + fn download(&self, dm: &DeviceMatrix) -> Matrix; // one D2H (applies perm) + + // in-place device kernels over a persistent buffer: + fn panel_factor(&self, dm, c, b, r, perm, out_L, out_pivcols) -> pr; // §5 + fn gemm_xor_into(&self, dm, L, pivot_rows, c_plus_b, r, pr); // §4.4, in place + // (gemm_xor_into wraps the existing wgmma.b1 kernel but writes C by XOR + // into an existing device region instead of alloc+dtoh) +} +``` + +Key deltas from today's kernel: +- **Persistent allocation** for `M`; kernels take device pointers, not host + slices. Reuse the *compute* core of `matmul_b1.cu` unchanged; change only the + host glue (no per-call `htod`/`dtoh`/`alloc_zeros`). +- **Fused epilogue: XOR-accumulate C into a target region** rather than writing a + fresh C and copying it back — saves an allocation and a D2H per panel and does + the `M[:,c+b:] ^= G` update for free inside the GEMM store. +- The `wgmma.b1` op is AND+popc→`s32`; the panel bit-pack/`n256` fragment layout + already solved in Phase 4 (`af251a5`) is reused for `L`/`U`. + +This is the bulk of the *new* GPU work. The CPU/algorithm side (§4) is +implementable and testable first (§9) against the existing AVX GEMM. + +--- + +## 7. Alternative: recursive PLE (most GEMM-heavy) + +Instead of a fixed panel width, split the column range in two and recurse +(`A = [A_L | A_R]`): PLE-factor `A_L`, update `A_R` by the resulting transform +(a GEMM), PLE-factor the trailing part of `A_R`, and combine the permutations. +Base case: a ~64-wide panel via §5(1). This is M4RI's `_mzd_pluq` / the +Dumas–Pernet–Sultan CUP decomposition. Pros: the panel term shrinks to another +level of GEMM, giving the sub-cubic `O(m·n·R^{ω−2})` shape and maximal tensor-core +occupancy. Cons: permutation bookkeeping (two-sided, rank-profile tracking) and a +triangular-solve-shaped update (`TRSM`), all as device kernels — materially more +to get right and to validate. **Recommendation:** ship the iterative blocked +scheme (§4) first — it already puts ~all flops in GEMM and is far easier to prove +correct — then graft recursion onto the *panel* (§5(2)) and, if profiling +warrants, onto the whole column split. + +--- + +## 8. Rank-deficiency specifics (the `R ≈ m/2` regime) + +1. **Work already ∝ R** (§3): the GEMM inner dimension is the pivots actually + found; half-rank ⇒ ~half the flops, automatically. +2. **Active-row compaction.** As columns are consumed, rows that are zero across + all remaining columns are dead weight in the GEMM's `m` dimension. Periodically + partition `perm` so the GEMM operates on `~R` active rows instead of `m`. + "Is this row now zero in `[c, n)`?" is an OR-reduce over a contiguous limb + range — a **row op**, cheap and coalesced. Compacting is a `perm` partition, + no byte movement (§4.3). For `R = m/2` this trims the trailing GEMM's tall + dimension toward `R`, compounding with point 1. +3. **Skip empty panels cheaply.** A panel with `pr = 0` (no pivots) does no GEMM + and no update — the sweep just advances `c`. Wide zero regions cost only their + `find-first` scans. + +--- + +## 9. Validation without a GPU (do this first) + +The **algorithm** (§4) is independent of the device; only the *residency* is +GPU-specific. So it can be built and proven correct entirely on CPU: + +- Implement §4 in `fp` against the existing `&Matrix * &Matrix` (AVX-512 tiled + GEMM) as the Step-B primitive, with a CPU panel factorization for Step A. +- **proptest** the result against `Matrix::row_reduce` for equality of the RREF + *and* the pivot list, over random `p=2` matrices across the shape grid already + used in `blas/mod.rs` tests, plus deliberately rank-deficient generators + (random rank `≤ min(m,n)/2`, duplicated/zero rows, zero columns). +- Add a criterion bench (mirroring the migrated algebra suite) measuring the + M4RI-vs-blocked crossover as a function of `(m, n, R, b)` to pick a default `b`. + +Once green on CPU, the GPU work (§6) is a *residency/perf* port with a +bit-exactness oracle already in hand — the CPU blocked result is the reference, +and the GPU path must match it bit-for-bit (the same discipline `fp-cuda` used +for the GEMM: CPU↔GPU bit-equality at every size). + +Crucially, this de-risks the expensive GPU effort: the design is falsifiable on a +laptop before a single device kernel is written. + +--- + +## 10. Concrete work plan + +| Phase | Where | Deliverable | Status | +|---|---|---|---| +| 0 | `fp` | This document | ✅ done | +| 1 | `fp` | CPU blocked forward-echelon (§4) + back-sub (§4.6) vs `row_reduce`; proptest + rank-deficient generators | ✅ done (`blas3.rs`) | +| 1b | `fp` | Block the back-substitution (§4.6) into the same `X·U` GEMM shape (right-to-left over pivot rows) | ✅ done (§10.3) | +| 1c | `fp` | Limb-wise panel factorization + back-sub gather (drop the per-bit `entry()` accessor) | ✅ done (§10.4) | +| 2 | `fp` | criterion bench; choose default `b`; measure M4RI crossover | gated by 1 | +| 3 | `fp-cuda` | Device-resident `DeviceMatrix` + persistent-buffer glue; `gemm_xor_into` fused epilogue (§6) | gated by 1,1b | +| 4 | `fp-cuda` | `panel_factor` kernel §5(1) (b=64) + virtual `perm` | gated by 3 | +| 5 | `fp-cuda` | Wire §4 to run entirely on device; bit-exact vs CPU blocked oracle | gated by 3,4 | +| 6 | `fp-cuda` | Active-row compaction (§8.2); panel-level GEMM §5(2) if profiled | gated by 5 | +| 7 | `fp` | Dispatch: `row_reduce` picks GPU path above a size/rank threshold, else M4RI | gated by 5 | + +Phase 1 is landed and proptested against `row_reduce`. Phase 1b (blocking the +back-substitution) and Phase 2 (the bench) need no GPU and come next. Phases 3–6 +are the residency port that turns the two constraints of §1 into the speedup. + +### 10.1 Phase 2 bench results (CPU, `benches/reduce_blas3.rs`) + +Measured on an AVX-512 host (so the GEMM is the tiled AVX kernel, not a scalar +fallback), half-rank inputs (`rank = n/2`, the target regime), criterion +median: + +| n | `row_reduce` (M4RI) | `row_reduce_blas3` | ratio | +|------|--------------------:|-------------------:|------:| +| 512 | 274 µs | 3.28 ms | 12× | +| 1024 | 941 µs | 14.8 ms | 16× | +| 2048 | 4.62 ms | 64.3 ms | 14× | +| 4096 | 27.2 ms | 307 ms | 11× | + +Panel-width sweep at `n = 2048` (half-rank): **64 → 1024 all land at 64–69 ms**, +i.e. within noise of each other. + +**Thread scaling** (half-rank `n = 2048`, `--features concurrent`, so the GEMM's +`maybe_rayon::join` is live; the earlier table above was built with `concurrent` +*off*, i.e. a serial GEMM — a methodology caveat): + +| threads | `row_reduce` (M4RI) | `row_reduce_blas3` | +|---------|--------------------:|-------------------:| +| 1 | 4.68 ms | 68.6 ms | +| 4 | 4.77 ms (**flat**) | 60.8 ms (**1.13×**)| + +This is the crux of *why* the approach matters and *why* this prototype does not +yet show it. M4RI is a **sequential dependency chain** — build the `2^k` table, +reduce, advance — so it is flat in the core count and cannot be sped up by +throwing cores at it. `L·U` GEMM is **data-parallel** and is the whole reason +the GPU kernel exists. But blas3 only scales 1.13× here because the GEMM is a +**minority** of its runtime; the majority is serial work this prototype left +naive, and Amdahl's law caps the speedup at `1/serial_fraction`. + +Which serial parts can actually be parallelized, and how: + +- **Trailing GEMM** — already parallel via `maybe_rayon` (needs `concurrent`). +- **Back-substitution** — blocking it into the mirror `L·U` GEMM (Phase 1b) + turns `O(R²·n)` serial row ops into a data-parallel GEMM. Biggest single lever + for the parallel fraction. +- **Panel below-row elimination** — embarrassingly parallel across rows *in + principle*, but only `≈ b/64` limbs of work per row per pivot column, so a + per-column `rayon` fork/join is swamped by its own overhead. It parallelizes + only if the whole panel is factored by a coarse limb-wise kernel (§5), not row + by row. On CPU the bigger win is just making it limb-wise (drop the per-bit + `entry()` accessor) to shrink the serial fraction; it stays hard to parallelize + cheaply. +- **Pivot search** — a `find-first` reduction; parallelizable but fine-grained, + same caveat. + +Sober extrapolation: even fully blocked, on **4 cores** blas3 will not cross +M4RI at these *small* sizes — the crossover needs either many-core CPUs or the +GPU's thousands of lanes, which is exactly the regime the design targets. The +value of Phase 1b + a limb-wise panel on CPU is to raise the parallel fraction so +the *scaling trend* is visible (and to serve as the GPU oracle), not to win on a +4-core box. + +### 10.2 Large-matrix trend — the regime that matters + +`examples/reduce_scaling.rs`, release, half-rank, one-shot per size, 4-core +AVX-512 host, 4 threads: + +| n | M4RI | blas3 | ratio | M4RI vs prev | blas3 vs prev | +|-------|--------:|--------:|-------:|-------------:|--------------:| +| 4096 | 28 ms | 299 ms | 10.6× | — | — | +| 8192 | 224 ms | 1.12 s | 5.0× | 8.0× | 3.7× | +| 16384 | 1.92 s | 8.16 s | 4.2× | 8.6× | 7.3× | +| 32768 | 24.8 s | 76.0 s | **3.07×** | **12.9×** | 9.3× | + +Two effects both favour blas3 as `n` grows, and both are visible here: + +1. **The ratio collapses monotonically** (10.6× → 3.07×). blas3's GEMM amortizes + better on larger operands while its fixed per-`entry()` panel overhead becomes + a vanishing fraction. +2. **M4RI degrades superlinearly at 32768** — the step is 12.9×, well past the + ~8× that `n³` predicts, because the 134 MB matrix overflows cache and M4RI's + table method turns memory-bound. blas3's *tiled* GEMM tracks `n³` (9.3×) — it + was built for the memory hierarchy. So large matrices punish M4RI twice: no + parallelism **and** worse locality. + +Extrapolating the ratio (~0.72× per doubling in the last step) puts the 4-core +crossover in the few-hundred-thousand range — and the real workload, 100 000 +rows, is ~27× more work than 2^15, squarely into that regime. On the GPU +(thousands of lanes, HBM bandwidth) the crossover moves down by orders of +magnitude. This is the quantitative case for the whole approach: **M4RI is a +sequential, cache-bound dead-end at scale; the GEMM path is not.** + +Caveats: single-shot (not averaged); blas3 is still slower than M4RI at every +size *measured on 4 cores*; and its serial `O(R²·n)` back-substitution (Phase 1b) +is an increasing drag — blocking it would bend the blas3 curve further down. + +### 10.3 Phase 1b: blocked back-substitution, re-measured + +After blocking the back-substitution into `X·U` GEMMs (§4.6, same harness, +4-core AVX-512 host): + +| n | M4RI | blas3 pre-1b | blas3 **1b** | ratio pre-1b | ratio **1b** | +|-------|--------:|-------------:|-------------:|-------------:|-------------:| +| 4096 | 28 ms | 299 ms | 207 ms | 10.6× | 7.3× | +| 8192 | 217 ms | 1.12 s | 1.30 s | 5.0× | 6.0× ⚠ | +| 16384 | 1.93 s | 8.16 s | **5.48 s** | 4.2× | **2.8×** | +| 32768 | 25.8 s | 76.0 s | **57.2 s** | 3.07× | **2.22×** | + +blas3 thread scaling (1→4 threads), pre-1b → 1b: + +| n | pre-1b | **1b** | +|-------|-------:|-------:| +| 4096 | 1.10× | 1.75× | +| 8192 | 1.72× | 1.23× ⚠ | +| 16384 | 1.23× | **1.89×** | + +The blocked back-substitution did what it was meant to at the sizes where it +matters: 16384 dropped 8.16 s → 5.48 s and its 4-core scaling rose 1.23× → 1.89× +(≈ 63 % parallel fraction by Amdahl), and 32768 dropped 76 s → 57 s with the ratio +falling 3.07× → 2.22×. The `⚠` at 8192 (blas3 slightly *slower*, scaling *lower*) +is single-shot noise — it sits between two points that both improved, and these +are un-averaged one-shot runs. The ratio still collapses with size (7.3× → 2.22×); +extrapolating the 1b ratio (~0.78× per doubling in the last step) puts the 4-core +crossover near `n ≈ 2.5 · 10⁵`, and 100 000 rows lands around ratio ≈ 1.5× — close +enough that a modest many-core CPU crosses it, and a GPU crosses it far earlier. + +Remaining serial drag (the next lever): the panel factorization's per-`entry()` +scan and the back-sub's `X` column-gather. Making the panel limb-wise (§5) is the +CPU follow-up; on the GPU it becomes the on-device panel kernel. + +### 10.4 Phase 1c: limb-wise panel, re-measured + +Replacing the per-bit `entry()`/`add_basis_element()` accessors in the panel +pivot search + forward elimination (and the back-sub `X` gather) with direct +limb reads/writes. Same harness, half-rank, 4-core AVX-512 host, 4 threads. +Full three-stage progression, blas3 time and (ratio to M4RI): + +| n | M4RI | pre-1b | +1b (§10.3) | +limb-wise | +|-------|--------:|--------------:|--------------:|--------------:| +| 4096 | 28 ms | 299 ms (10.6×)| 207 ms (7.3×) | 209 ms (7.5×) | +| 8192 | 224 ms | 1.12 s (5.0×) | 1.30 s (6.0×) | **1.02 s (4.6×)** | +| 16384 | ~2 s | 8.16 s (4.2×) | 5.48 s (2.8×) | **4.00 s (1.8×)** | +| 32768 | ~26–31 s| 76.0 s (3.07×)| 57.2 s (2.22×)| **45.9 s (1.46×)** | + +The two optimizations compound where it counts: at 16384, blas3 fell **8.16 s → +4.00 s** (2.0×) and the ratio **4.2× → 1.8×**; at 32768, **76 s → 45.9 s** (1.66×) +and **3.07× → 1.46×**. The limb-wise pass helps *more* at large `n` (8192/16384/ +32768) than at 4096, exactly because the `O(m·n)` panel scan — which the accessor +overhead sat on top of — is a growing fraction there. + +blas3 4-core scaling (1→4 threads) with limb-wise: 4096 1.18×, 8192 1.27×, 16384 +1.62×. It rises with `n` (more parallel GEMM exposed); it reads lower than the 1b +row above only because limb-wise cut the *serial* panel time, shrinking the gap +between the 1- and 4-thread runs while dropping both. + +**Where the crossover now sits.** The 4-thread ratio is 1.46× at 2^15 and falling +~0.81× per doubling; extrapolated, blas3 crosses M4RI on **just 4 cores around +n ≈ 1.2 · 10⁵** — i.e. right at the real 100 000-row workload. Two 5-line +constant-factor rewrites moved the CPU crossover from "few × 10⁵" down onto the +target size. And this is still the regime the GPU is *far* better at: `wgmma.b1` +is orders of magnitude more parallel than 4 CPU cores, and the panel/gather +serial work moves on-device. The trend line is now unambiguous — the GEMM path +overtakes M4RI exactly where the problem gets big. + +### 10.5 How the reading changed — small vs large matrices + +The first bench (§10.1, sizes ≤ 4096) read as "~10× slower, not GEMM-bound, a +GPU-only technique — don't route CPU `row_reduce` here." That conclusion was an +artefact of **small** matrices: there the `O(m·n)` per-`entry()` panel scan and +the serial back-sub dominate a GEMM that is itself small and overhead-heavy, so +M4RI's cache-friendly table wins comfortably. + +The large-matrix trend (§10.2) plus the two constant-factor fixes (§10.3–10.4) +overturned it. As `n` grows the GEMM fraction rises, M4RI goes memory-bound +(§10.2), and blocking the back-sub + making the panel limb-wise strip the serial +overhead — so the ratio falls to **1.46× at 2^15 on 4 cores**, extrapolating to a +crossover around **n ≈ 1.2 · 10⁵**. So: + +- The approach is **not** GPU-only. It already closes to near-parity with M4RI on + a 4-core CPU at the target size, and the crossover moves down with more cores. +- Phase 7's dispatch should therefore be a **size/rank threshold**, not + "GPU-only": above the crossover (and/or when many cores are available) route to + `row_reduce_blas3`; below it, keep M4RI. +- The GPU remains where it wins *decisively* — `wgmma.b1` is far more parallel + than 4 CPU cores and the panel/gather work moves on-device — but the CPU path + is now a real speedup at scale, not just a correctness oracle. + +The through-line is unchanged and now quantified: **M4RI is sequential and +cache-bound; the GEMM path is parallel and hierarchy-friendly, so it wins exactly +where the problem gets big — which is the regime that matters here.** + +--- + +## 11. Summary + +- Do **not** loop over `matmul_b1_raw`: it re-marshals the matrix every panel and + makes PCIe the ceiling (§1.1). Keep the matrix in **one device buffer** and run + a sequence of in-place kernels; bus traffic stays ≈ 2× the matrix (§6). +- Confine every **column** access to the narrow panel; express all wide work as + **GEMM** or contiguous **row-XOR**; never move columns; make row swaps virtual + via `perm` so the 1.25 GB buffer never shuffles (§1.2, §4.3). +- Use right-looking **blocked Gauss–Jordan** (PLE family): panel-factor → + capture multipliers `L` → one trailing **GEMM** `L·U` XOR-ed in place (§4). + Total cost `O(m·n·R)`, GEMM-dominated, **∝ rank** — the rank-½ workload is + ~2× cheaper for free (§3), with compaction trimming the GEMM's tall dimension + toward `R` (§8). +- Prove it on CPU against `row_reduce` before touching the GPU (§9). + +### References + +- M. Albrecht, G. Bard, C. Pernet — *Efficient Dense Gaussian Elimination over + the Finite Field with Two Elements* (arXiv:1111.6549). The M4RI block-iterative + **PLE**; the basis for §4/§7. +- J.-G. Dumas, C. Pernet, Z. Sultan — *Rank-profile revealing Gaussian + elimination and the CUP matrix decomposition* (arXiv:1112.5717). +- *Fast matrix decomposition in F₂* (arXiv:1209.5198). +- J.-G. Dumas, C. Pernet — *Computational linear algebra over finite fields* + (arXiv:1204.3735) — survey; reduction of elimination to matmul. +- *A Study on Optimization of Sparse and Dense Linear System Solver over GF(2) + on GPUs*, Springer (LNCS) — GPU bit-packed blocked elimination. diff --git a/ext/crates/fp/Cargo.toml b/ext/crates/fp/Cargo.toml index b0123aba05..52dd296e2a 100644 --- a/ext/crates/fp/Cargo.toml +++ b/ext/crates/fp/Cargo.toml @@ -21,6 +21,12 @@ serde_json = "1.0.141" maybe-rayon = { path = "../maybe-rayon" } query = { path = "../query" } +# GPU backend (Hopper wgmma.b1). Optional and off by default: gated behind the +# `gpu` feature. `fp-cuda`'s library is fp-agnostic (raw limbs), so this +# dependency does not form a cycle. Building it requires nvcc at build time (a +# stub PTX is emitted otherwise) and a Hopper GPU + driver at runtime. +fp-cuda = { path = "../fp-cuda", optional = true } + [dev-dependencies] # We use the proptest harness for our own tests fp = { path = ".", default-features = false, features = ["proptest"] } @@ -38,6 +44,8 @@ build_const = "0.2.2" default = ["odd-primes"] concurrent = ["maybe-rayon/concurrent"] odd-primes = [] +# Dispatch large p=2 matrix products to the Hopper GPU backend (`fp-cuda`). +gpu = ["dep:fp-cuda"] [[bench]] name = "mul" @@ -47,6 +55,10 @@ harness = false name = "reduce" harness = false +[[bench]] +name = "reduce_blas3" +harness = false + [[bench]] name = "smallfq" harness = false diff --git a/ext/crates/fp/benches/reduce_blas3.rs b/ext/crates/fp/benches/reduce_blas3.rs new file mode 100644 index 0000000000..02a2ab0e6a --- /dev/null +++ b/ext/crates/fp/benches/reduce_blas3.rs @@ -0,0 +1,99 @@ +//! M4RI (`row_reduce`) vs blocked GEMM-based (`row_reduce_blas3`) row reduction +//! over F₂, on full-rank and (the target workload) rank-deficient matrices, plus +//! a panel-width sweep to inform the default block. +//! +//! Note: on CPU the GEMM path is the AVX-512 tiled kernel (or the scalar +//! fallback if the host lacks AVX-512), so the crossover measured here is a +//! lower bound on the eventual GPU (`wgmma.b1`) speedup. + +use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; +use fp::{matrix::Matrix, prime::TWO}; +use pprof::criterion::{Output, PProfProfiler}; +use rand::Rng; + +fn random_matrix(rows: usize, cols: usize) -> Matrix { + let mut rng = rand::rng(); + Matrix::from_vec( + TWO, + &(0..rows) + .map(|_| (0..cols).map(|_| rng.random_range(0..2)).collect()) + .collect::>>(), + ) +} + +/// An `n × n` F₂ matrix of rank at most `rank`, as the product of a random +/// `n × rank` and `rank × n` matrix (built with the fast GEMM). This models the +/// highly rank-deficient inputs the algorithm targets. +fn rank_deficient_matrix(n: usize, rank: usize) -> Matrix { + let a = random_matrix(n, rank); + let b = random_matrix(rank, n); + &a * &b +} + +fn full_rank(c: &mut Criterion) { + let mut group = c.benchmark_group("row_reduce_f2_full_rank"); + for n in [512usize, 1024, 2048] { + group.bench_with_input(BenchmarkId::new("m4ri", n), &n, |bch, &n| { + bch.iter_batched_ref( + || random_matrix(n, n), + |m| m.row_reduce(), + BatchSize::LargeInput, + ) + }); + group.bench_with_input(BenchmarkId::new("blas3", n), &n, |bch, &n| { + bch.iter_batched_ref( + || random_matrix(n, n), + |m| m.row_reduce_blas3(), + BatchSize::LargeInput, + ) + }); + } + group.finish(); +} + +fn rank_deficient(c: &mut Criterion) { + let mut group = c.benchmark_group("row_reduce_f2_half_rank"); + for n in [512usize, 1024, 2048, 4096] { + let rank = n / 2; + group.bench_with_input(BenchmarkId::new("m4ri", n), &n, |bch, &n| { + bch.iter_batched_ref( + || rank_deficient_matrix(n, rank), + |m| m.row_reduce(), + BatchSize::LargeInput, + ) + }); + group.bench_with_input(BenchmarkId::new("blas3", n), &n, |bch, &n| { + bch.iter_batched_ref( + || rank_deficient_matrix(n, rank), + |m| m.row_reduce_blas3(), + BatchSize::LargeInput, + ) + }); + } + group.finish(); +} + +/// Panel-width sweep at a fixed rank-deficient size, to pick the default block. +fn block_sweep(c: &mut Criterion) { + let mut group = c.benchmark_group("row_reduce_f2_block_sweep_2048"); + let n = 2048; + let rank = n / 2; + for block in [64usize, 128, 256, 512, 1024] { + group.bench_with_input(BenchmarkId::from_parameter(block), &block, |bch, &block| { + bch.iter_batched_ref( + || rank_deficient_matrix(n, rank), + |m| m.row_reduce_blas3_block(block), + BatchSize::LargeInput, + ) + }); + } + group.finish(); +} + +criterion_group! { + name = benches; + config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = full_rank, rank_deficient, block_sweep +} + +criterion_main!(benches); diff --git a/ext/crates/fp/examples/reduce_scaling.rs b/ext/crates/fp/examples/reduce_scaling.rs new file mode 100644 index 0000000000..56abf9d922 --- /dev/null +++ b/ext/crates/fp/examples/reduce_scaling.rs @@ -0,0 +1,68 @@ +//! One-shot large-matrix timing of M4RI (`row_reduce`) vs blocked GEMM +//! (`row_reduce_blas3`) over F₂ on half-rank inputs, printed per size. Run with +//! `--features concurrent` and set `RAYON_NUM_THREADS` to vary the core count. +//! +//! ```sh +//! RAYON_NUM_THREADS=4 cargo run --release -p fp --features concurrent \ +//! --example reduce_scaling -- 4096 8192 16384 +//! ``` + +use std::time::Instant; + +use fp::{matrix::Matrix, prime::TWO}; +use rand::Rng; + +fn random_matrix(rows: usize, cols: usize) -> Matrix { + let mut rng = rand::rng(); + Matrix::from_vec( + TWO, + &(0..rows) + .map(|_| (0..cols).map(|_| rng.random_range(0..2)).collect()) + .collect::>>(), + ) +} + +/// `n × n` of rank ≤ `rank`, as a product (built with the fast GEMM). +fn rank_deficient(n: usize, rank: usize) -> Matrix { + &random_matrix(n, rank) * &random_matrix(rank, n) +} + +fn main() { + let sizes: Vec = std::env::args() + .skip(1) + .filter_map(|a| a.parse().ok()) + .collect(); + let sizes = if sizes.is_empty() { + vec![4096, 8192, 16384] + } else { + sizes + }; + let threads = std::env::var("RAYON_NUM_THREADS").unwrap_or_else(|_| "default".to_string()); + + println!( + "{:>7} {:>10} {:>10} {:>7} (threads={threads})", + "n", "m4ri", "blas3", "ratio" + ); + for n in sizes { + let base = rank_deficient(n, n / 2); + + let mut a = base.clone(); + let t = Instant::now(); + let r1 = a.row_reduce(); + let m4ri = t.elapsed(); + drop(a); + + let mut b = base.clone(); + let t = Instant::now(); + let r2 = b.row_reduce_blas3(); + let blas3 = t.elapsed(); + + assert_eq!(r1, r2, "rank mismatch at n={n}"); + println!( + "{n:>7} {:>10.3?} {:>10.3?} {:>6.2}x (rank {r1})", + m4ri, + blas3, + blas3.as_secs_f64() / m4ri.as_secs_f64(), + ); + } +} diff --git a/ext/crates/fp/proptest-regressions/matrix/blas3.txt b/ext/crates/fp/proptest-regressions/matrix/blas3.txt new file mode 100644 index 0000000000..198441ddbf --- /dev/null +++ b/ext/crates/fp/proptest-regressions/matrix/blas3.txt @@ -0,0 +1,9 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc ba39cc57d4d1f24634cc46da015ea15fe5025b90a462dd518236949d89b9ccf5 # shrinks to m = [ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0], [1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1], [0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1], [1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0] ] +cc 58a61f9d0ae09b72b4e0996db539d7e2a8a296e2005189580a9edfdbcbc2b561 # shrinks to m = [ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0], [1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0], [1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1], [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1], [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1], [1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1], [1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1], [1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0], [0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1], [1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0], [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1], [0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0], [1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0], [0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1], [0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0], [1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0], [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0], [0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0], [1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0], [1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0], [1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1], [1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0], [1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0], [1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1], [0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1], [1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0], [0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1], [1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1], [0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0], [1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1], [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1], [1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0], [0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1], [0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1], [0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1], [0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1], [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0], [0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0], [0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0], [0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0], [1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0] ] +cc 01895ccac7d106fb818520ec1d1ea140e82a30b103bb0b0f0809b2a815504a7c # shrinks to m = [ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0], [0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1] ] diff --git a/ext/crates/fp/src/blas/cuda.rs b/ext/crates/fp/src/blas/cuda.rs new file mode 100644 index 0000000000..0fba7154ec --- /dev/null +++ b/ext/crates/fp/src/blas/cuda.rs @@ -0,0 +1,153 @@ +//! GPU dispatch for F₂ matrix multiplication (Hopper `wgmma.b1`). +//! +//! Compiled only under the `gpu` feature. [`try_mul`] is consulted by +//! `<&Matrix as Mul>::mul` before the CPU BLAS path: for large enough `p = 2` +//! products it converts the operands to the raw row-major limb layout +//! `fp-cuda` expects, runs the kernel, and rebuilds a [`Matrix`]. Anything that +//! makes the GPU path unavailable or unsuitable — no device, a launch error, or +//! a below-threshold size — returns `None`, and the caller falls back to the +//! (bit-identical) CPU kernel. +//! +//! Tuning knobs (environment variables, read once): +//! - `FP_CUDA_DISABLE` — set to any value to force the CPU path. +//! - `FP_CUDA_THRESHOLD` — minimum of `m`, `k`, `n` (in bits) below which the +//! CPU path is used. Defaults to 2048; the GPU only wins once the kernel work +//! dwarfs the H2D/D2H + TMA-layout marshalling, which dominates small sizes. + +use std::sync::{Mutex, OnceLock}; + +use fp_cuda::GpuContext; + +use crate::{matrix::Matrix, prime::TWO}; + +/// Smallest `min(m, k, n)` for which we attempt the GPU matmul. Below this the +/// host marshalling (bit-repack into TMA tiles + copies) costs more than it saves. +const DEFAULT_THRESHOLD: usize = 2048; + +/// Smallest `min(rows, cols)` for which we attempt the GPU row reduction. Higher +/// than the matmul threshold: a full reduction is many dependent panel steps, not +/// one GEMM, so its CPU crossover is later. Re-validated on an H200 post- +/// optimization (half-rank square, device incl. upload/reduce vs M4RI +/// `row_reduce`): GPU is 0.57× at n=4096 (a loss) and 1.57× at n=8192 (a win), +/// so the crossover sits just below 8192. The small-n crossover is bound by fixed +/// launch/transfer overhead, not the trailing GEMM, so the recent throughput wins +/// (which scale with n²) did not move it. Measured against single-thread M4RI; +/// the concurrent CPU path is faster, which only pushes the crossover up — so +/// 8192 is the safe floor. Override with `FP_CUDA_RR_THRESHOLD`. +const DEFAULT_RR_THRESHOLD: usize = 8192; + +fn threshold() -> usize { + std::env::var("FP_CUDA_THRESHOLD") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_THRESHOLD) +} + +fn rr_threshold() -> usize { + std::env::var("FP_CUDA_RR_THRESHOLD") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_RR_THRESHOLD) +} + +/// The process-wide GPU context, created lazily on first use. `None` if no +/// usable device is present (no driver, no Hopper GPU, or the kernel PTX is the +/// nvcc-absent build stub). Wrapped in a `Mutex` because a single CUDA context +/// serialises submission anyway and `GpuContext` is not shared concurrently. +fn context() -> Option<&'static Mutex> { + static GPU: OnceLock>> = OnceLock::new(); + GPU.get_or_init(|| { + if std::env::var_os("FP_CUDA_DISABLE").is_some() { + return None; + } + match GpuContext::new(0) { + Ok(ctx) => Some(Mutex::new(ctx)), + Err(_) => None, + } + }) + .as_ref() +} + +/// Row-major, K-major `u64` limbs — the exact layout `fp_cuda::matmul_b1_raw` +/// expects (`rows × columns.div_ceil(64)` limbs, no inter-row padding). Uses +/// `Matrix::to_bytes`, which already strips the physical row stride. +fn to_limbs(m: &Matrix) -> Vec { + let stride = m.columns().div_ceil(64); + let mut bytes = Vec::with_capacity(m.rows() * stride * 8); + m.to_bytes(&mut bytes).expect("Vec writes never fail"); + let (chunks, _) = bytes.as_chunks::<8>(); + chunks.iter().map(|&c| u64::from_le_bytes(c)).collect() +} + +/// Try to compute `a · b` on the GPU. Returns `None` (and the caller uses the +/// CPU path) if the GPU is unavailable, the product is below the size +/// threshold, or the launch fails. The result is bit-identical to the CPU path. +/// +/// Assumes `a.prime() == b.prime() == 2` and `a.columns() == b.rows()` — the +/// same preconditions the caller has already checked. +pub(super) fn try_mul(a: &Matrix, b: &Matrix) -> Option { + debug_assert_eq!(a.prime(), TWO); + debug_assert_eq!(b.prime(), TWO); + debug_assert_eq!(a.columns(), b.rows()); + + let (m, k, n) = (a.rows(), a.columns(), b.columns()); + let t = threshold(); + if m < t || k < t || n < t { + return None; + } + + let ctx = context()?; + let a_limbs = to_limbs(a); + let b_limbs = to_limbs(b); + + let c = { + let guard = ctx.lock().ok()?; + fp_cuda::matmul_b1_raw(&guard, &a_limbs, m, k, &b_limbs, n).ok()? + }; + Some(Matrix::from_data(TWO, m, n, c)) +} + +/// Try to row-reduce `m` to RREF on the GPU, in place. Returns `Some(rank)` and +/// leaves `m` in the same canonical reduced form `Matrix::row_reduce` produces +/// (pivot rows at the top in column order, zeros below, `pivots` set); returns +/// `None` — and the caller uses the CPU M4RI path — if the GPU is unavailable, +/// below threshold, or a launch fails. The result is bit-identical to the CPU +/// path (validated in `fp-cuda`'s `row_reduce_demo`). +/// +/// Assumes `m.prime() == 2` (the caller has checked). +pub(crate) fn try_row_reduce(m: &mut Matrix) -> Option { + debug_assert_eq!(m.prime(), TWO); + let (rows, cols) = (m.rows(), m.columns()); + let t = rr_threshold(); + if rows < t || cols < t { + return None; + } + let ctx = context()?; + + let stride = cols.div_ceil(64); + let limbs = to_limbs(m); + + let (dev_limbs, perm, r, pivot_cols) = { + let gpu = ctx.lock().ok()?; + let mut dm = gpu.upload(&limbs, rows, cols).ok()?; + let (perm, r, pivot_cols) = gpu.row_reduce_dev(&mut dm).ok()?; + let dev_limbs = gpu.download(&dm).ok()?; + let perm = gpu.download_u32(&perm).ok()?; + (dev_limbs, perm, r, pivot_cols) + }; + + // Materialize the canonical RREF: pivot k (column pivot_cols[k], ascending) + // at row k, taken from device row perm[k]; rows [r, rows) zero. + let mut out = vec![0u64; rows * stride]; + for k in 0..r { + let src = perm[k] as usize * stride; + out[k * stride..k * stride + stride].copy_from_slice(&dev_limbs[src..src + stride]); + } + *m = Matrix::from_data(TWO, rows, cols, out); + m.initialize_pivots(); + let piv = m.pivots_mut(); + for (k, &q) in pivot_cols.iter().enumerate() { + piv[q] = k as isize; + } + Some(r) +} diff --git a/ext/crates/fp/src/blas/mod.rs b/ext/crates/fp/src/blas/mod.rs index b5487628a8..76f25ebd89 100644 --- a/ext/crates/fp/src/blas/mod.rs +++ b/ext/crates/fp/src/blas/mod.rs @@ -31,6 +31,9 @@ use crate::matrix::Matrix; pub mod block; pub mod tile; +#[cfg(feature = "gpu")] +pub(crate) mod cuda; + impl std::ops::Mul for &Matrix { type Output = Matrix; @@ -44,6 +47,10 @@ impl std::ops::Mul for &Matrix { { // Can use optimized BLAS operations (matrix rows are padded to multiple of 64) // TODO: Use different block sizes and loop orders based on the size of the matrices + #[cfg(feature = "gpu")] + if let Some(result) = cuda::try_mul(self, rhs) { + return result; + } self.fast_mul_concurrent(rhs) } else { // Use naive multiplication for: diff --git a/ext/crates/fp/src/matrix/blas3.rs b/ext/crates/fp/src/matrix/blas3.rs new file mode 100644 index 0000000000..31ea7450ee --- /dev/null +++ b/ext/crates/fp/src/matrix/blas3.rs @@ -0,0 +1,467 @@ +//! BLAS3 (GEMM-based) row reduction over F₂. +//! +//! This is the CPU-resident Phase 1 of the plan in `BLAS3-ROW-REDUCTION.md`: a +//! right-looking blocked Gauss–Jordan reduction whose dominant work — the +//! trailing-submatrix update — is a single F₂ matrix product per column panel, +//! dispatched through `<&Matrix as Mul>::mul` (the AVX-512 tiled kernel today, +//! the Hopper `wgmma.b1` kernel once `fp-cuda` is wired in). It produces exactly +//! the same reduced row echelon form and pivot list as [`Matrix::row_reduce`], +//! which is what the proptests at the bottom assert. +//! +//! # Shape of the algorithm +//! +//! Sweep column panels `[c, c+b)` left to right, keeping a running pivot count +//! `r` (the finished pivot rows live at the top, in column order). +//! +//! * **Panel factorization** (`[c, c+b)`): Gauss–Jordan on the panel, but with +//! the row operations restricted to the panel's limbs. This is the only place +//! we touch individual columns, and it is narrow (`b` a small multiple of 64). +//! As each pivot column `q_k` is cleared from a row `j`, the cleared bit is the +//! multiplier `L[j][k]`; because the pivot rows are kept mutually reduced, that +//! bit equals the pristine `A[j, q_k]` and the `L[j, ·]` are independent. +//! * **Trailing update**: `M[:, c+b:] ^= L · U`, where `U` is the pivot rows' +//! trailing part. One F₂ GEMM. Because `c+b` is a multiple of 64 the trailing +//! region starts at a limb boundary, so `U` is a contiguous limb slice and the +//! XOR of the product back into `M` is a whole-limb, row-parallel operation. +//! +//! Total cost is `O(m · n · R)` with `R` the rank, GEMM-dominated and +//! proportional to rank — so the highly rank-deficient matrices this targets are +//! cheaper for free. + +use crate::{limb::Limb, matrix::Matrix, prime::TWO}; + +/// Default panel width in columns. A multiple of 64; wide enough that the +/// trailing GEMM's inner (`k`) dimension is a healthy number of pivots, narrow +/// enough that the panel factorization stays a lower-order term. +const DEFAULT_BLOCK_COLS: usize = 256; + +impl Matrix { + /// GEMM-based reduction to reduced row echelon form; F₂ only. + /// + /// Bit-for-bit identical result and pivots to [`Matrix::row_reduce`]. Falls + /// back to [`Matrix::row_reduce`] for `p ≠ 2`. + /// + /// # Returns + /// The rank (number of non-zero rows after reduction), like `row_reduce`. + pub fn row_reduce_blas3(&mut self) -> usize { + self.row_reduce_blas3_block(DEFAULT_BLOCK_COLS) + } + + /// [`Matrix::row_reduce_blas3`] with an explicit panel width (rounded down to + /// a multiple of 64, and at least 64). Exposed for benchmarking the crossover. + pub fn row_reduce_blas3_block(&mut self, block_cols: usize) -> usize { + if self.prime() != 2 { + return self.row_reduce(); + } + + let m = self.rows(); + let n = self.columns(); + self.initialize_pivots(); + if m == 0 || n == 0 { + return 0; + } + + // Panels are whole limbs. Columns are stored padded to a limb boundary + // (`stride` limbs, bits past `n` are zero), so we can reason in limbs and + // treat the final partial limb as a full one WLOG — the padding columns + // are zero, never become pivots, and cost nothing. Only the pivot search, + // which indexes real columns, is capped at `n`. + let stride = self.stride(); // limbs per row + let bl = (block_cols.max(64) & !63) / 64; // panel width in limbs + + // Reused scratch for a pivot row's panel limbs, so the inner elimination + // loop is a plain limb XOR with no per-row re-borrow. + let mut piv_panel = vec![0 as Limb; bl]; + + let mut r = 0usize; // pivots found so far; pivot rows sit at [0, r) + let mut limb_lo = 0usize; // current panel's first limb + while limb_lo < stride { + let limb_hi = (limb_lo + bl).min(stride); // panel's end limb (exclusive) + let col_lo = limb_lo * 64; + let col_hi = (limb_hi * 64).min(n); // last real column of the panel + let r_start = r; + + // ---- Step A: panel factorization on columns [col_lo, col_hi) ---- + // Row ops are restricted to the panel's limbs [limb_lo, limb_hi); the + // trailing columns [limb_hi, stride) are fixed up by the GEMM below. + // + // Multiplier bits, one column per pivot found. Only *deferred* rows + // (rows below the pivots) get an entry; the pivot rows are made exact + // by the promotion step instead. + let mut l = Self::new(TWO, m, col_hi - col_lo); + let mut pr = 0usize; // pivots found in this panel + + let panel_w = limb_hi - limb_lo; + for q in col_lo..col_hi { + let piv_row = r_start + pr; + let qlimb = q / 64; // absolute limb of column q (in [limb_lo, limb_hi)) + let qbit = q % 64; + + // Pivot search (raw-limb): first row in [piv_row, m) with bit q + // set. Rows [piv_row, m) already had this panel's earlier pivot + // columns cleared, so a non-zero here is a genuine pivot. + let mut found = None; + { + let data = self.data(); + for i in piv_row..m { + if (data[i * stride + qlimb] >> qbit) & 1 == 1 { + found = Some(i); + break; + } + } + } + let Some(i) = found else { continue }; + + if i != piv_row { + self.swap_rows(i, piv_row); + // `l` is indexed by current row position, so it must track the + // same swap: a row moved up here may already carry deferred + // multiplier bits from earlier pivots in this panel. + l.swap_rows(i, piv_row); + } + + // Promote `piv_row` to an exact echelon pivot row: realize its + // deferred trailing by replaying this panel's earlier pivots into + // its trailing columns. Those earlier pivots are stable — forward + // elimination never reduces a pivot by a *later* one — so their + // trailing is final and this catch-up is correct. Its panel limbs + // were already reduced while it was a below row. + { + let lstride = l.stride(); + for kp in 0..pr { + if (l.data()[piv_row * lstride + kp / 64] >> (kp % 64)) & 1 == 1 { + xor_limb_range(self, piv_row, r_start + kp, limb_hi, stride); + } + } + } + zero_row(&mut l, piv_row); + + // Forward elimination (raw-limb): clear column q from the rows + // *below* the pivot only, recording the multiplier and touching + // just the panel limbs (trailing deferred to the GEMM). Rows above + // are left for the back-substitution pass — reducing them here + // would destabilize the pivot trailings the promotion relies on. + { + let piv_base = piv_row * stride; + piv_panel[..panel_w] + .copy_from_slice(&self.data()[piv_base + limb_lo..piv_base + limb_hi]); + } + { + let lstride = l.stride(); + let lword = pr / 64; + let lmask: Limb = 1 << (pr % 64); + let ldata = l.data_mut(); + let data = self.data_mut(); + for j in piv_row + 1..m { + let jbase = j * stride; + if (data[jbase + qlimb] >> qbit) & 1 == 1 { + ldata[j * lstride + lword] |= lmask; + for t in 0..panel_w { + data[jbase + limb_lo + t] ^= piv_panel[t]; + } + } + } + } + + self.pivots_mut()[q] = piv_row as isize; + pr += 1; + } + r = r_start + pr; + + // ---- Step B: trailing update M[:, limb_hi:] ^= L · U ---- + let trailing_limbs = stride - limb_hi; + if pr > 0 && trailing_limbs > 0 { + let t = n - limb_hi * 64; // real trailing columns (> 0 here) + + // Trim L to its m × pr occupied columns so the GEMM's inner + // dimension is the pivots actually found, not the panel width. + let n_l = pr.div_ceil(64); + let mut l_trim = Self::new(TWO, m, pr); + { + let l_stride = l.stride(); + let lt_stride = l_trim.stride(); + let src = l.data(); + let dst = l_trim.data_mut(); + for j in 0..m { + dst[j * lt_stride..j * lt_stride + n_l] + .copy_from_slice(&src[j * l_stride..j * l_stride + n_l]); + } + } + + // U = pivot rows' trailing part, a contiguous limb slice. + let mut u = Self::new(TWO, pr, t); + { + let u_stride = u.stride(); // == trailing_limbs + debug_assert_eq!(u_stride, trailing_limbs); + let src = self.data(); + let dst = u.data_mut(); + for k in 0..pr { + let base = (r_start + k) * stride + limb_hi; + dst[k * u_stride..k * u_stride + trailing_limbs] + .copy_from_slice(&src[base..base + trailing_limbs]); + } + } + + // One F₂ GEMM (GPU / AVX-512 tiled kernel), then a whole-limb XOR + // of the m × t product into M's trailing columns. + let g = &l_trim * &u; + let g_stride = g.stride(); // == trailing_limbs + { + let src = g.data(); + let dst = self.data_mut(); + for j in 0..m { + let m_base = j * stride + limb_hi; + let g_base = j * g_stride; + for col in 0..trailing_limbs { + dst[m_base + col] ^= src[g_base + col]; + } + } + } + } + + limb_lo = limb_hi; + } + + // ---- Back-substitution: echelon form → reduced row echelon form ---- + // The forward sweep left the R = `r` pivot rows in echelon form (rows + // [0, r), pivot k in increasing column order, each already reduced by + // *earlier* pivots). We now clear the *later* pivot columns from the rows + // above them, blocked into the same `X·U` GEMM shape as the forward + // trailing update — the mirror image, walking pivot blocks right-to-left. + // + // For a block of pivots (rows [s, e)): (1) reduce the block to RREF among + // itself with a few full-width row ops, then (2) clear the block's pivot + // columns from every row above via one GEMM. Blocks to the right are done + // first, so a block's rows are already clear of every pivot column to + // their right when we reach them, and rows above are reduced by the block + // in one shot. Same O(R²·n) work as the naive sweep, but as GEMMs. + let pivot_cols: Vec = (0..n).filter(|&q| self.pivots()[q] >= 0).collect(); + debug_assert_eq!(pivot_cols.len(), r); + let bp = bl * 64; // pivots per back-substitution block + + let mut e = r; + while e > 0 { + let s = e.saturating_sub(bp); + let bp_eff = e - s; + + // (1) Reduce block rows [s, e) to RREF among themselves: clear each + // block pivot column from the earlier block rows. Processing high-to- + // low keeps every source row fully reduced before it is used. + for k in (s..e).rev() { + let qk = pivot_cols[k]; + for j in s..k { + if self.row(j).entry(qk) != 0 { + self.safe_row_op(j, k, 1); + } + } + } + + // (2) Clear the block's pivot columns from all rows above [0, s) with + // one GEMM: `M[0..s, :] ^= X · U`, where `X[j][i] = M[j, q_{s+i}]` and + // `U` is the (now RREF) block rows. The block rows are zero below + // column `q_s`, so the update starts at that limb boundary. + if s > 0 { + let start_limb = pivot_cols[s] / 64; + let trailing_limbs = stride - start_limb; + let width = n - start_limb * 64; + + // X = rows above, gathered at this block's pivot columns (s × + // bp_eff), raw-limb: read bit `q_{s+i}` of row j, set bit i of X. + let mut x = Self::new(TWO, s, bp_eff); + { + let x_stride = x.stride(); + let src = self.data(); + let dst = x.data_mut(); + for j in 0..s { + for i in 0..bp_eff { + let qc = pivot_cols[s + i]; + if (src[j * stride + qc / 64] >> (qc % 64)) & 1 == 1 { + dst[j * x_stride + i / 64] |= 1 << (i % 64); + } + } + } + } + + // U = block rows [s, e), limbs [start_limb, stride) (bp_eff × width). + let mut u = Self::new(TWO, bp_eff, width); + { + let u_stride = u.stride(); // == trailing_limbs + debug_assert_eq!(u_stride, trailing_limbs); + let src = self.data(); + let dst = u.data_mut(); + for i in 0..bp_eff { + let base = (s + i) * stride + start_limb; + dst[i * u_stride..i * u_stride + trailing_limbs] + .copy_from_slice(&src[base..base + trailing_limbs]); + } + } + + let g = &x * &u; + let g_stride = g.stride(); // == trailing_limbs + { + let src = g.data(); + let dst = self.data_mut(); + for j in 0..s { + let m_base = j * stride + start_limb; + let g_base = j * g_stride; + for col in 0..trailing_limbs { + dst[m_base + col] ^= src[g_base + col]; + } + } + } + } + + e = s; + } + + r + } +} + +/// Zero every limb of row `row` in matrix `m` (used to discharge a pivot row's +/// deferred multipliers once its trailing has been realized). +fn zero_row(m: &mut Matrix, row: usize) { + let stride = m.stride(); + let base = row * stride; + m.data_mut()[base..base + stride].fill(0); +} + +/// XOR the limbs `[lo, hi)` of row `src` into the same limbs of row `dst`. +/// `dst != src` required. Restricted to a limb range so it can touch a panel +/// without disturbing the trailing columns. +fn xor_limb_range(m: &mut Matrix, dst: usize, src: usize, lo: usize, hi: usize) { + debug_assert_ne!(dst, src); + let stride = m.stride(); + let d = dst * stride; + let s = src * stride; + let data = m.data_mut(); + // Split at the higher row's start so the two rows land in disjoint halves. + if d < s { + let (left, right) = data.split_at_mut(s); + for l in lo..hi { + left[d + l] ^= right[l]; + } + } else { + let (left, right) = data.split_at_mut(d); + for l in lo..hi { + right[l] ^= left[s + l]; + } + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use crate::{ + matrix::{Matrix, arbitrary::MatrixArbParams}, + prime::TWO, + }; + + /// Arbitrary F₂ matrices across a spread of shapes, including sizes above and + /// below the 32-row padding boundary and the 64-column limb boundary, and + /// wider than one default panel. + fn arb_matrix(max_dim: usize) -> impl Strategy { + (1usize..=max_dim, 1usize..=max_dim).prop_flat_map(|(rows, cols)| { + Matrix::arbitrary_with(MatrixArbParams { + p: Some(TWO), + rows: Just(rows).boxed(), + columns: Just(cols).boxed(), + }) + }) + } + + /// A deliberately rank-deficient F₂ matrix: `A · B` with a small shared + /// dimension `rank_bound`, so the product has rank at most `rank_bound`. + fn arb_low_rank() -> impl Strategy { + (40usize..300, 40usize..300, 1usize..30).prop_flat_map(|(rows, cols, rank_bound)| { + let a = Matrix::arbitrary_with(MatrixArbParams { + p: Some(TWO), + rows: Just(rows).boxed(), + columns: Just(rank_bound).boxed(), + }); + let b = Matrix::arbitrary_with(MatrixArbParams { + p: Some(TWO), + rows: Just(rank_bound).boxed(), + columns: Just(cols).boxed(), + }); + (a, b).prop_map(|(a, b)| &a * &b) + }) + } + + fn assert_matches_row_reduce(m: &Matrix, block: usize) -> Result<(), TestCaseError> { + let mut reference = m.clone(); + let ref_rank = reference.row_reduce(); + + let mut blocked = m.clone(); + let blocked_rank = blocked.row_reduce_blas3_block(block); + + prop_assert_eq!(blocked_rank, ref_rank, "rank mismatch"); + prop_assert_eq!(blocked.pivots(), reference.pivots(), "pivot mismatch"); + prop_assert_eq!(&blocked, &reference, "RREF mismatch"); + Ok(()) + } + + // Deterministic sweep over small shapes that straddle the 64-column limb + // boundary (so multiple panels and partial-limb trailings are exercised). + // On mismatch it prints the exact input, unlike a shrunk proptest dump. + #[test] + fn matches_row_reduce_deterministic() { + // xorshift RNG for reproducibility (no external rng dep needed here). + let mut state: u64 = 0x9E3779B97F4A7C15; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for rows in 1usize..12 { + for cols in 65usize..200 { + // multiple panels at block 64, various partial-limb trailings + for _ in 0..40 { + let input: Vec> = (0..rows) + .map(|_| (0..cols).map(|_| (next() & 1) as u32).collect()) + .collect(); + let base = Matrix::from_vec(TWO, &input); + let mut reference = base.clone(); + reference.row_reduce(); + let mut blocked = base.clone(); + blocked.row_reduce_blas3_block(64); + if blocked != reference || blocked.pivots() != reference.pivots() { + panic!( + "MISMATCH rows={rows} cols={cols}\ninput={input:?}\nexpected \ + pivots={:?}\n got pivots={:?}\nexpected=\n{reference}\n \ + got=\n{blocked}", + reference.pivots(), + blocked.pivots() + ); + } + } + } + } + } + + proptest! { + #[test] + fn matches_row_reduce_small(m in arb_matrix(80)) { + assert_matches_row_reduce(&m, 64)?; + } + + #[test] + fn matches_row_reduce_default_block(m in arb_matrix(200)) { + assert_matches_row_reduce(&m, 256)?; + } + + // A small block forces many panels, exercising the panel-to-panel + // invariant (trailing update of already-reduced pivot rows). + #[test] + fn matches_row_reduce_tiny_block(m in arb_matrix(200)) { + assert_matches_row_reduce(&m, 64)?; + } + + #[test] + fn matches_row_reduce_low_rank(m in arb_low_rank()) { + assert_matches_row_reduce(&m, 128)?; + } + } +} diff --git a/ext/crates/fp/src/matrix/matrix_inner.rs b/ext/crates/fp/src/matrix/matrix_inner.rs index 54b90924ad..99b014e631 100644 --- a/ext/crates/fp/src/matrix/matrix_inner.rs +++ b/ext/crates/fp/src/matrix/matrix_inner.rs @@ -673,6 +673,17 @@ impl Matrix { /// ``` pub fn row_reduce(&mut self) -> usize { let p = self.prime(); + + // For large p = 2 matrices, try the device-resident GPU reduction; it + // produces the identical canonical RREF + pivots. Falls back to the CPU + // M4RI path below when the GPU is unavailable or below threshold. + #[cfg(feature = "gpu")] + if p == 2 + && let Some(rank) = crate::blas::cuda::try_row_reduce(self) + { + return rank; + } + self.initialize_pivots(); let mut empty_rows = Vec::with_capacity(self.rows()); diff --git a/ext/crates/fp/src/matrix/mod.rs b/ext/crates/fp/src/matrix/mod.rs index 1ff0fda7b1..b681b981c8 100644 --- a/ext/crates/fp/src/matrix/mod.rs +++ b/ext/crates/fp/src/matrix/mod.rs @@ -1,5 +1,6 @@ // mod basis; mod affine; +mod blas3; mod matrix_inner; mod quasi_inverse; mod subquotient; diff --git a/ext/crates/fp/tests/cuda_dispatch.rs b/ext/crates/fp/tests/cuda_dispatch.rs new file mode 100644 index 0000000000..d339339277 --- /dev/null +++ b/ext/crates/fp/tests/cuda_dispatch.rs @@ -0,0 +1,97 @@ +//! GPU-dispatch correctness for `<&Matrix as Mul>::mul` under the `gpu` +//! feature. Only compiled when the feature is on; if no usable GPU is present +//! the dispatch transparently falls back to the CPU kernel, so this test still +//! passes (it just doesn't exercise the device). Run with `FP_CUDA_DEBUG=1` to +//! see the `[fp-cuda]` launch line and confirm the GPU path was taken. +#![cfg(feature = "gpu")] + +use fp::{matrix::Matrix, prime::TWO}; +use rand::Rng; + +fn random_matrix(rows: usize, cols: usize) -> Matrix { + let mut rng = rand::rng(); + let limbs = cols.div_ceil(64); + let data: Vec = (0..rows * limbs).map(|_| rng.random()).collect(); + Matrix::from_data(TWO, rows, cols, data) +} + +/// Well-formed 0/1 matrix (bits past the last column masked), of rank ≤ `rank` +/// when `rank > 0`, else full-random. +fn clean_matrix(rows: usize, cols: usize, rank: usize) -> Matrix { + let mut rng = rand::rng(); + let mut rand_vec = |r: usize, c: usize| -> Matrix { + let v: Vec> = (0..r) + .map(|_| (0..c).map(|_| rng.random::() as u32).collect()) + .collect(); + Matrix::from_vec(TWO, &v) + }; + if rank == 0 { + rand_vec(rows, cols) + } else { + &rand_vec(rows, rank) * &rand_vec(rank, cols) + } +} + +/// The dispatched product must be bit-identical to the CPU BLAS kernel. Sizes +/// are chosen above the default 2048 threshold so the GPU path is attempted. +#[test] +fn gpu_dispatch_matches_cpu() { + for &(m, k, n) in &[ + (2048, 2048, 2048), + (4096, 2048, 3072), + (3072, 4096, 2048), + // Non-tile-aligned dims (not multiples of the kernel's 192/128/1024 + // tiles; rows still pad to a multiple of 64 so dispatch fires): + // exercise edge masks, partial limbs, and raster tails. + (2049, 2051, 2053), + (3000, 2112, 4097), + ] { + let a = random_matrix(m, k); + let b = random_matrix(k, n); + + // `&a * &b` consults the GPU (feature on); `fast_mul_concurrent` is the + // reference CPU kernel. They must agree bit-for-bit. + let dispatched = &a * &b; + let reference = a.fast_mul_concurrent(&b); + + assert_eq!( + dispatched, reference, + "GPU/CPU mismatch at {m}x{k} * {k}x{n}" + ); + } +} + +/// The dispatched `row_reduce` (GPU, feature on) must be bit-identical to the CPU +/// BLAS3 reducer — RREF, rank, and pivots. The production row-reduce threshold is +/// 8192 (below that the CPU wins); force it down here so these small, fast test +/// shapes still exercise the GPU path. This test uses `FP_CUDA_RR_THRESHOLD`, +/// distinct from the matmul test's `FP_CUDA_THRESHOLD`, so the two don't collide. +#[test] +fn gpu_row_reduce_matches_cpu() { + // SAFETY: set once at the start of the test, before any threshold() read. + unsafe { std::env::set_var("FP_CUDA_RR_THRESHOLD", "2048") }; + for &(rows, cols, rank) in &[ + (2048, 2048, 0), + (4096, 2560, 0), + (2048, 3000, 0), + (3000, 2048, 500), // rank-deficient + ] { + let base = clean_matrix(rows, cols, rank); + + let mut gpu = base.clone(); + let rank_gpu = gpu.row_reduce(); // GPU dispatch (feature on, above threshold) + let mut cpu = base.clone(); + let rank_cpu = cpu.row_reduce_blas3(); // CPU oracle, never dispatches + + assert_eq!( + rank_gpu, rank_cpu, + "rank mismatch at {rows}x{cols} rank={rank}" + ); + assert_eq!( + gpu.pivots(), + cpu.pivots(), + "pivot mismatch at {rows}x{cols}" + ); + assert_eq!(gpu, cpu, "RREF mismatch at {rows}x{cols} rank={rank}"); + } +} diff --git a/ext/flake.nix b/ext/flake.nix index 35e0d02fc8..0858136926 100644 --- a/ext/flake.nix +++ b/ext/flake.nix @@ -7,7 +7,23 @@ outputs = {super, ...}: super.flake-utils.lib.eachDefaultSystem (system: let - pkgs = import super.nixpkgs {inherit system;}; + # Allow CUDA (unfree in nixpkgs). Scoped to the CUDA / NVIDIA prefix so + # we don't accidentally unfree-allow anything else. nixpkgs splits the + # toolkit into many sub-derivations (cuda_nvcc, cuda_cudart, cuda-merged, + # cuda_cuobjdump, libcublas, ...) — listing them individually is whack- + # a-mole, so we match by prefix. + pkgs = import super.nixpkgs { + inherit system; + config.allowUnfreePredicate = pkg: + let + lib = super.nixpkgs.lib; + name = lib.getName pkg; + in + lib.hasPrefix "cuda" name + || lib.hasPrefix "libcu" name + || lib.hasPrefix "libnv" name + || lib.hasPrefix "libnpp" name; + }; pythonEnv = pkgs.python3.withPackages (ps: [ ps.black @@ -27,6 +43,13 @@ pkgs.perf ] ++ super.defaultPackages.devTools.${system}; + + # CUDA toolkit for building fp-cuda's Hopper wgmma.b1 kernel: nvcc + headers + # at build time. Kept out of `commonPackages` (and the default shell) so + # contributors and the `apps.test`/CI closure don't fetch the multi-GB unfree + # CUDA tree for the opt-in backend. cudarc dlopens libcuda at runtime, so + # only running — not building the Rust — needs the host driver. + cudatoolkit = pkgs.cudaPackages.cudatoolkit; in { devShells.default = pkgs.mkShell { packages = commonPackages; @@ -35,6 +58,17 @@ ''; }; + # GPU dev shell: `nix develop .#gpu`. Adds the CUDA toolkit (nvcc + headers) + # and points the loader at both it and the host driver's libcuda. + devShells.gpu = pkgs.mkShell { + packages = commonPackages ++ [cudatoolkit]; + shellHook = '' + export RUST_LOG=info + export CUDA_PATH="${cudatoolkit}" + export LD_LIBRARY_PATH="${cudatoolkit}/lib:/run/opengl-driver/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + ''; + }; + apps.test = { type = "app"; packages = commonPackages;