From 32b00b05b94d2a93118a11060911c89ac7a502c0 Mon Sep 17 00:00:00 2001 From: 0z5a <192209249+0z5a@users.noreply.github.com> Date: Sun, 13 Sep 2026 11:14:59 +0800 Subject: [PATCH] Add SmolLM2 tile128 schedule preset and generation evidence Signed-off-by: 0z5a <192209249+0z5a@users.noreply.github.com> --- docs/smollm2_tile128.md | 75 +++++++++++ docs/smollm2_tile128_results.json | 194 +++++++++++++++++++++++++++++ examples/smollm2_gemv_tile128.json | 7 ++ 3 files changed, 276 insertions(+) create mode 100644 docs/smollm2_tile128.md create mode 100644 docs/smollm2_tile128_results.json create mode 100644 examples/smollm2_gemv_tile128.json diff --git a/docs/smollm2_tile128.md b/docs/smollm2_tile128.md new file mode 100644 index 0000000..6419c5d --- /dev/null +++ b/docs/smollm2_tile128.md @@ -0,0 +1,75 @@ +# SmolLM2 GEMV tile128 schedule experiment + +`examples/smollm2_gemv_tile128.json` selects the existing `N_tile=128` schedule +knob. The configuration is opt-in; the default lowerer, CUDA VM, ABI, oracle and +kernel knobs are unchanged. + +On the measured SmolLM2 workload, default tiling produced 5,286 tasks per step +and tile128 produced 1,716, with 848 buffers and 38,912 bytes of dynamic shared +memory in both arms. A wider tile can reduce schedule/table setup work while +reducing GPU parallelism. The result below measures complete generation; it +is not a GPU-only GEMV claim. + +## Using the configuration + +The existing evaluator accepts the preset: + +```bash +uv run amk eval HuggingFaceTB/SmolLM2-135M-Instruct --gpu rtx5090 \ + --config examples/smollm2_gemv_tile128.json +``` + +That command evaluates one forward using the registered target. It does not +reproduce the complete-generation experiment or substitute an RTX 5090 target +for a PRO 4000. The archived generation runner reads the live SM120 device +limits and passes the same `ScheduleConfig` to the unmodified lowerer at each +step. `generate` currently passes `config=None`, so reproducing the full request +requires that external adapter; this PR adds no new generation API. + +## Recorded complete-generation experiment + +Frozen upstream: `a514bbc20a03bbf698a17443f8f14a27a617fc10`. +Model: `HuggingFaceTB/SmolLM2-135M-Instruct`, revision +`12fd25f77366fa6b3b4b768ec3050bf629380bac`, all 30 layers and 134,515,008 +parameters, FP32 weights/activations, no quantization. Six checkpoint file hashes +are included in the evidence. Runtime: Python 3.12, Torch 2.13.0+cu130, +Transformers 5.16.1, CUDA 13.0, one RTX PRO 4000 Blackwell 24 GB (SM120). + +Four independent processes ran default/tile128/tile128/default, with independent +first-build directories. Context lengths were 8 and 64 tokens; each request +produced 8 new tokens. All 344 step-level logits comparisons passed the existing +FP32 `atol=rtol=1e-4` oracle against both CPU ReferenceVM and full eager. +All 64 correctness-stage generated tokens matched exactly. Only after both +workloads passed did each process perform one warmup and three timed requests +per context; timed and warmup sequences also matched. + +| Context | Process pair | Default median seconds | Tile128 median seconds | Latency reduction | +|---|---|---:|---:|---:| +| 8 | A0/P1 | 3.486624 | 1.778946 | 48.98% | +| 8 | A3/P2 | 3.515794 | 1.755405 | 50.07% | +| 64 | A0/P1 | 16.170751 | 7.911105 | 51.08% | +| 64 | A3/P2 | 15.953324 | 7.895794 | 50.51% | + +Allocator peak: 1,320,648,704 bytes in each process. The two process-pair +geometric means are 49.53% and 50.79% latency reduction for contexts 8 and 64. +The baseline last/first median changes are +0.84% and -1.34% respectively. +These are exploratory observations from two process pairs, not a confidence +interval or independently confirmed stable speedup. + +Timing includes per-step lowering/validation, VM and table construction, weight +H2D, KV transfer and sampling from a CPU-resident model. It excludes disk model +loading, first CUDA compilation and correctness instrumentation. The existing +path reconstructs the VM each step; this is not GPU-resident steady-state +decode, concurrent service throughput, CUDA Graph replay, or a comparison with +cuBLAS/vLLM. + +The preset passed the experiment's advancement rule: every context/process +pair improved by at least 1%. Further independent validation is required before +changing a default. A separate `cpa_stages=2` candidate regressed at context64 +and is not included. + +[All timings, per-step comparisons, checkpoint hashes and the original runner](https://gist.github.com/0z5a/7b6a65fb1899b27411706bb402201016) +are published together. The archive documents the exact directory layout and +commands for replaying all four processes. The compact checked-in +[results](smollm2_tile128_results.json) retain every recorded request time and +original receipt hashes. No GPU test was rerun for publication. diff --git a/docs/smollm2_tile128_results.json b/docs/smollm2_tile128_results.json new file mode 100644 index 0000000..fa18af9 --- /dev/null +++ b/docs/smollm2_tile128_results.json @@ -0,0 +1,194 @@ +{ + "upstream_commit": "a514bbc20a03bbf698a17443f8f14a27a617fc10", + "claim_level": "EXPLORATORY_2_PROCESS_PAIRS", + "metric": "synchronized full generate wall seconds from resident CPU model, including lowering/validation/H2D/table/KV/sampling; excludes disk checkpoint loading and completed compilation", + "model": { + "repo_id": "HuggingFaceTB/SmolLM2-135M-Instruct", + "revision": "12fd25f77366fa6b3b4b768ec3050bf629380bac", + "files": { + "config.json": { + "bytes": 861, + "sha256": "8eb740e8bbe4cff95ea7b4588d17a2432deb16e8075bc5828ff7ba9be94d982a" + }, + "generation_config.json": { + "bytes": 132, + "sha256": "87b916edaaab66b3899b9d0dd0752727dff6666686da0504d89ae0a6e055a013" + }, + "model.safetensors": { + "bytes": 269060552, + "sha256": "5af571cbf074e6d21a03528d2330792e532ca608f24ac70a143f6b369968ab8c" + }, + "tokenizer_config.json": { + "bytes": 3764, + "sha256": "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df" + }, + "special_tokens_map.json": { + "bytes": 655, + "sha256": "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52" + }, + "tokenizer.json": { + "bytes": 2104556, + "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c" + } + } + }, + "order": [ + "A", + "P", + "P", + "A" + ], + "precision": { + "weights": "float32", + "activations": "float32", + "quantization": null, + "atol": 0.0001, + "rtol": 0.0001 + }, + "gpu": "NVIDIA RTX PRO 4000 Blackwell", + "torch": "2.13.0+cu130", + "cuda": "13.0", + "arms": [ + { + "index": 0, + "tile": 0, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 3.4639218139927834, + 3.533638576976955, + 3.4866243698634207 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 3.4866243698634207 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 16.039617849048227, + 16.17075055697933, + 16.200428175041452 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 16.17075055697933 + } + ], + "allocator_peak_bytes": 1320648704, + "source_receipt_sha256": "948604e4d6f107582494ee3f358477ba9f55fae0e9dd076dae9f71a221be5c15" + }, + { + "index": 1, + "tile": 128, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 1.778946174075827, + 1.8109971762169152, + 1.765465283067897 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 1.778946174075827 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 7.911105460021645, + 8.040637442842126, + 7.805065699853003 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 7.911105460021645 + } + ], + "allocator_peak_bytes": 1319820288, + "source_receipt_sha256": "6b3a39c0c1b28d22c70e2c97c4c63f026556222935a8eec332605bf584fff8ca" + }, + { + "index": 2, + "tile": 128, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 1.7565167911816388, + 1.7502881770487875, + 1.7554047601297498 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 1.7554047601297498 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 7.8609512688126415, + 8.023758386960253, + 7.8957938810344785 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 7.8957938810344785 + } + ], + "allocator_peak_bytes": 1319820288, + "source_receipt_sha256": "f2e1f950386da0a8de66000b226ea64847e6c6ffb4a395cbe1880c66149363ad" + }, + { + "index": 3, + "tile": 0, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 3.5770872849971056, + 3.515794370090589, + 3.4626008081249893 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 3.515794370090589 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 15.95332440501079, + 16.115130202146247, + 15.89765602699481 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 15.95332440501079 + } + ], + "allocator_peak_bytes": 1320648704, + "source_receipt_sha256": "45b11107487573ed07ae7fade072ed2f96686dd77e2b4bc9a19bcaf47db75da2" + } + ], + "validation": { + "step_comparisons": 344, + "all_cases_pass": true, + "all_generated_tokens_match": true + } +} diff --git a/examples/smollm2_gemv_tile128.json b/examples/smollm2_gemv_tile128.json new file mode 100644 index 0000000..9da9f77 --- /dev/null +++ b/examples/smollm2_gemv_tile128.json @@ -0,0 +1,7 @@ +{ + "tiling": { + "gemv": { + "N_tile": 128 + } + } +}