From 7250bc0d2f69f8fe78acb280528b623a3c6f9b16 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:54:10 +0000 Subject: [PATCH 1/3] feat(site): add measured gpt2-small beside gpt2-large refs Persist Prism-protocol openai-community/gpt2 metrics on /references and dual GPT-2 columns in the HF top-model README template. --- crates/prism-registry/src/hf.rs | 220 ++++++++++++++++++++-------- crates/site-api/src/handlers.rs | 9 +- crates/site-api/src/prism_enrich.rs | 133 +++++++++++------ docs/SITE_API.md | 4 +- 4 files changed, 253 insertions(+), 113 deletions(-) diff --git a/crates/prism-registry/src/hf.rs b/crates/prism-registry/src/hf.rs index ca270550f..79e7c70c3 100644 --- a/crates/prism-registry/src/hf.rs +++ b/crates/prism-registry/src/hf.rs @@ -33,19 +33,33 @@ const REGULAR_FILE_MAX: usize = 5 * 1024 * 1024; const BANNER_URL: &str = "https://github.com/BaseIntelligence/base/raw/main/assets/banner.jpg"; /// GPT-2 Large (774M) Prism-protocol public-pack reference (eval-only). -const GPT2_LABEL: &str = "GPT-2 Large (774M)"; -const GPT2_PARAMS_M: f64 = 774.0; -const GPT2_BPB: f64 = 4.163_851_322_121_356_4; -const GPT2_HELLASWAG: f64 = 0.395; -const GPT2_ARC_EASY: f64 = 0.28; -const GPT2_ARC_CHALLENGE: f64 = 0.28; -const GPT2_PIQA: f64 = 0.69; -const GPT2_WINOGRANDE: f64 = 0.545; -const GPT2_BOOLQ: f64 = 0.64; -/// Prism-protocol Lium 1×5090 public pack (not OpenAI paper LAMBADA 60.12%). -const GPT2_LAMBADA: f64 = 0.985; -const GPT2_OPENBOOKQA: f64 = 0.335; -const GPT2_SOURCE: &str = "https://huggingface.co/gpt2-large"; +const GPT2_LARGE_LABEL: &str = "GPT-2 Large (774M)"; +const GPT2_LARGE_PARAMS_M: f64 = 774.0; +const GPT2_LARGE_BPB: f64 = 4.163_851_322_121_356_4; +const GPT2_LARGE_HELLASWAG: f64 = 0.395; +const GPT2_LARGE_ARC_EASY: f64 = 0.28; +const GPT2_LARGE_ARC_CHALLENGE: f64 = 0.28; +const GPT2_LARGE_PIQA: f64 = 0.69; +const GPT2_LARGE_WINOGRANDE: f64 = 0.545; +const GPT2_LARGE_BOOLQ: f64 = 0.64; +const GPT2_LARGE_LAMBADA: f64 = 0.985; +const GPT2_LARGE_OPENBOOKQA: f64 = 0.335; +const GPT2_LARGE_SOURCE: &str = "https://huggingface.co/gpt2-large"; + +/// GPT-2 Small (124M) Prism-protocol public-pack reference (eval-only). +const GPT2_SMALL_LABEL: &str = "GPT-2 (124M)"; +const GPT2_SMALL_PARAMS_M: f64 = 124.4; +const GPT2_SMALL_BPB: f64 = 4.759_478_148_923_918; +const GPT2_SMALL_HELLASWAG: f64 = 0.355; +const GPT2_SMALL_ARC_EASY: f64 = 0.245; +const GPT2_SMALL_ARC_CHALLENGE: f64 = 0.24; +const GPT2_SMALL_PIQA: f64 = 0.585; +const GPT2_SMALL_WINOGRANDE: f64 = 0.515; +const GPT2_SMALL_BOOLQ: f64 = 0.575; +const GPT2_SMALL_LAMBADA: f64 = 0.97; +const GPT2_SMALL_OPENBOOKQA: f64 = 0.32; +const GPT2_SMALL_SOURCE: &str = "https://huggingface.co/openai-community/gpt2"; + /// HuggingFace Hub publisher (token never `Debug`/`Display`'d). pub struct HfTopModelPublisher { @@ -502,9 +516,10 @@ fn hub_readme(req: &TopModelRequest, arch: &str, has_ckpt: bool) -> String { "Weights were not parked on the master for this champion — sources/config only." }; let params_cell = params_m.map_or_else(|| "—".into(), |p| format!("{p:.1}M")); - let gpt2_params = format!("{GPT2_PARAMS_M:.0}M"); - let params_vs = match params_m { - Some(p) if p > 0.0 => format!("{:.2}× vs {GPT2_LABEL}", GPT2_PARAMS_M / p), + let gpt2_large_params = format!("{GPT2_LARGE_PARAMS_M:.0}M"); + let gpt2_small_params = format!("{GPT2_SMALL_PARAMS_M:.0}M"); + let params_vs_large = match params_m { + Some(p) if p > 0.0 => format!("{:.2}× vs {GPT2_LARGE_LABEL}", GPT2_LARGE_PARAMS_M / p), _ => "—".into(), }; let tflops_cell = tflops.map_or_else(|| "—".into(), |t| format!("{t:.1} TFLOPS (est.)")); @@ -523,47 +538,93 @@ fn hub_readme(req: &TopModelRequest, arch: &str, has_ckpt: bool) -> String { let _ = writeln!(out, "![BASE Banner]({BANNER_URL})"); out.push('\n'); out.push_str("

PRISM top architecture

\n\n"); - out.push_str("

Global-best miner architecture on Base PRISM — benchmarks vs GPT-2 Large

\n\n"); + out.push_str("

Global-best miner architecture on Base PRISM — benchmarks vs GPT-2 / GPT-2 Large

\n\n"); out.push_str("\n\n---\n\n"); - out.push_str("## Benchmarks vs GPT-2 Large\n\n"); - out.push_str("Prism-protocol **public** eval pack. Accuracy: **↑ higher better**. BPB: **↓ lower better**. "); + out.push_str("## Benchmarks vs GPT-2 (Prism-protocol)\n\n"); + out.push_str("Prism-protocol **public** eval pack (1×RTX 5090). Accuracy: **↑ higher better**. BPB: **↓ lower better**. "); let _ = writeln!( out, - "Reference: [{GPT2_LABEL}]({GPT2_SOURCE}) (eval-only; not a miner train)." + "References: [{GPT2_SMALL_LABEL}]({GPT2_SMALL_SOURCE}) · [{GPT2_LARGE_LABEL}]({GPT2_LARGE_SOURCE}) (eval-only; not miner trains)." ); out.push('\n'); - out.push_str("| Metric | This model | GPT-2 Large | Δ | vs GPT-2 Large |\n"); - out.push_str("|---|---:|---:|---:|:---|\n"); - out.push_str(&bench_row_lower("Val BPB (G1)", Some(req.bpb), GPT2_BPB, 4)); - for (name, ours, theirs) in [ - ("HellaSwag", benches.hellaswag, GPT2_HELLASWAG), - ("ARC-Easy", benches.arc_easy, GPT2_ARC_EASY), - ("ARC-Challenge", benches.arc_challenge, GPT2_ARC_CHALLENGE), - ("PIQA", benches.piqa, GPT2_PIQA), - ("WinoGrande", benches.winogrande, GPT2_WINOGRANDE), - ("BoolQ", benches.boolq, GPT2_BOOLQ), - ("LAMBADA", benches.lambada, GPT2_LAMBADA), - ("OpenBookQA", benches.openbookqa, GPT2_OPENBOOKQA), + out.push_str("| Metric | This model | GPT-2 | GPT-2 Large | vs GPT-2 | vs GPT-2 Large |\n"); + out.push_str("|---|---:|---:|---:|:---|:---|\n"); + out.push_str(&bench_row_lower_dual( + "Val BPB (G1)", + Some(req.bpb), + GPT2_SMALL_BPB, + GPT2_LARGE_BPB, + 4, + )); + for (name, ours, small, large) in [ + ( + "HellaSwag", + benches.hellaswag, + GPT2_SMALL_HELLASWAG, + GPT2_LARGE_HELLASWAG, + ), + ( + "ARC-Easy", + benches.arc_easy, + GPT2_SMALL_ARC_EASY, + GPT2_LARGE_ARC_EASY, + ), + ( + "ARC-Challenge", + benches.arc_challenge, + GPT2_SMALL_ARC_CHALLENGE, + GPT2_LARGE_ARC_CHALLENGE, + ), + ("PIQA", benches.piqa, GPT2_SMALL_PIQA, GPT2_LARGE_PIQA), + ( + "WinoGrande", + benches.winogrande, + GPT2_SMALL_WINOGRANDE, + GPT2_LARGE_WINOGRANDE, + ), + ("BoolQ", benches.boolq, GPT2_SMALL_BOOLQ, GPT2_LARGE_BOOLQ), + ( + "LAMBADA", + benches.lambada, + GPT2_SMALL_LAMBADA, + GPT2_LARGE_LAMBADA, + ), + ( + "OpenBookQA", + benches.openbookqa, + GPT2_SMALL_OPENBOOKQA, + GPT2_LARGE_OPENBOOKQA, + ), ] { - out.push_str(&bench_row_higher(name, ours, theirs, 3)); + out.push_str(&bench_row_higher_dual(name, ours, small, large, 3)); } out.push_str("\n### Compute notes\n\n"); - out.push_str("| | This model | GPT-2 Large |\n|---|---|---|\n"); - let _ = writeln!(out, "| Parameters | {params_cell} | {gpt2_params} |"); - let _ = writeln!(out, "| Size vs reference | {params_vs} | 1× |"); + out.push_str("| | This model | GPT-2 | GPT-2 Large |\n|---|---|---|---|\n"); + let _ = writeln!( + out, + "| Parameters | {params_cell} | {gpt2_small_params} | {gpt2_large_params} |" + ); + let _ = writeln!( + out, + "| Size vs Large | {params_vs_large} | {:.2}× | 1× |", + GPT2_LARGE_PARAMS_M / GPT2_SMALL_PARAMS_M + ); + let _ = writeln!( + out, + "| Train tokens | {tokens_cell} | _(eval-only)_ | _(eval-only)_ |" + ); let _ = writeln!( out, - "| Train tokens | {tokens_cell} | _(eval-only reference)_ |" + "| Wall clock | {wall_cell} | _(eval-only)_ | _(eval-only)_ |" ); let _ = writeln!( out, - "| Wall clock | {wall_cell} | _(eval-only reference)_ |" + "| Sustained train throughput | {tflops_cell} | n/a | n/a |" ); let _ = writeln!( out, - "| Sustained train throughput | {tflops_cell} | n/a (no Prism train) |" + "| GPU (harness) | `{gpu}` | 1×RTX 5090 (eval) | 1×RTX 5090 (eval) |" ); - let _ = writeln!(out, "| GPU (harness) | `{gpu}` | 1×RTX 5090 (eval) |"); out.push_str( "\nThroughput ≈ `6 × N × D / wall` TFLOPS (dense transformer train FLOPs rule of thumb).\n\n", ); @@ -711,44 +772,75 @@ fn train_tokens(metrics: Option<&serde_json::Value>) -> Option { .or_else(|| metric_u64(metrics, &["tokens_seen"]).filter(|&t| t > 10_000)) } -fn bench_row_higher(name: &str, ours: Option, base: f64, digits: usize) -> String { + + +fn verdict_higher(delta: f64) -> (&'static str, &'static str) { + if delta.abs() < 1e-9 { + ("=", "tie") + } else if delta > 0.0 { + ("↑", "✓ better") + } else { + ("↓", "worse") + } +} + +fn verdict_lower(delta: f64) -> (&'static str, &'static str) { + if delta.abs() < 1e-9 { + ("=", "tie") + } else if delta < 0.0 { + ("↓", "✓ better") + } else { + ("↑", "worse") + } +} + +fn bench_row_higher_dual( + name: &str, + ours: Option, + small: f64, + large: f64, + digits: usize, +) -> String { match ours { Some(v) => { - let delta = v - base; - let (arrow, verdict) = if (delta).abs() < 1e-9 { - ("=", "tie") - } else if delta > 0.0 { - ("↑", "✓ better") - } else { - ("↓", "worse") - }; + let ds = v - small; + let dl = v - large; + let (as_, vs) = verdict_higher(ds); + let (al, vl) = verdict_higher(dl); format!( - "| {name} | {v:.digits$} | {base:.digits$} | {arrow} {delta:+.digits$} | {verdict} |\n" + "| {name} | {v:.digits$} | {small:.digits$} | {large:.digits$} | {as_} {ds:+.digits$} {vs} | {al} {dl:+.digits$} {vl} |\n" ) } - None => format!("| {name} | — | {base:.digits$} | — | _(missing)_ |\n"), + None => format!( + "| {name} | — | {small:.digits$} | {large:.digits$} | _(missing)_ | _(missing)_ |\n" + ), } } -fn bench_row_lower(name: &str, ours: Option, base: f64, digits: usize) -> String { +fn bench_row_lower_dual( + name: &str, + ours: Option, + small: f64, + large: f64, + digits: usize, +) -> String { match ours { Some(v) => { - let delta = v - base; - let (arrow, verdict) = if (delta).abs() < 1e-9 { - ("=", "tie") - } else if delta < 0.0 { - ("↓", "✓ better") - } else { - ("↑", "worse") - }; + let ds = v - small; + let dl = v - large; + let (as_, vs) = verdict_lower(ds); + let (al, vl) = verdict_lower(dl); format!( - "| {name} | {v:.digits$} | {base:.digits$} | {arrow} {delta:+.digits$} | {verdict} |\n" + "| {name} | {v:.digits$} | {small:.digits$} | {large:.digits$} | {as_} {ds:+.digits$} {vs} | {al} {dl:+.digits$} {vl} |\n" ) } - None => format!("| {name} | — | {base:.digits$} | — | _(missing)_ |\n"), + None => format!( + "| {name} | — | {small:.digits$} | {large:.digits$} | _(missing)_ | _(missing)_ |\n" + ), } } + const CONFIGURATION_PRISM_PY: &str = r#" """Prism custom-arch Hub config (trust_remote_code).""" @@ -1014,7 +1106,9 @@ mod tests { #[test] fn readme_benchmarks_first_vs_gpt2_large() { let md = hub_readme(&req(), "arch_hf", true); - assert!(md.contains("Benchmarks vs GPT-2 Large"), "{md}"); + assert!(md.contains("Benchmarks vs GPT-2 (Prism-protocol)"), "{md}"); + assert!(md.contains("GPT-2 Large"), "{md}"); + assert!(md.contains("| GPT-2 |"), "{md}"); assert!(md.contains(BANNER_URL), "{md}"); assert!(md.contains("HellaSwag"), "{md}"); assert!(md.contains("✓ better") || md.contains("worse"), "{md}"); diff --git a/crates/site-api/src/handlers.rs b/crates/site-api/src/handlers.rs index 432a7fe7f..3512fd3e1 100644 --- a/crates/site-api/src/handlers.rs +++ b/crates/site-api/src/handlers.rs @@ -1363,14 +1363,21 @@ mod tests { let (s, v) = call(app.clone(), "/v1/site/arenas/prism/references").await; assert_eq!(s, StatusCode::OK, "{v}"); - assert_eq!(v.as_array().unwrap().len(), 1); + assert_eq!(v.as_array().unwrap().len(), 2); assert_eq!(v[0]["id"], "gpt2-large-774m"); + assert_eq!(v[1]["id"], "gpt2-small-124m"); assert_eq!(v[0]["paramsM"], 774.0); + assert!((v[1]["paramsM"].as_f64().unwrap() - 124.4).abs() < 1e-9); assert!(v[0]["bpb"].as_f64().unwrap() > 1.0); + assert!(v[1]["bpb"].as_f64().unwrap() > 1.0); assert!(v[0]["disclaimer"] .as_str() .unwrap() .contains("Prism-protocol")); + assert!(v[1]["disclaimer"] + .as_str() + .unwrap() + .contains("Prism-protocol")); let (s, v) = call(app, "/v1/site/arenas/prism/submissions/nope").await; assert_eq!(s, StatusCode::NOT_FOUND, "{v}"); diff --git a/crates/site-api/src/prism_enrich.rs b/crates/site-api/src/prism_enrich.rs index c9d4dbb1a..96ce20aac 100644 --- a/crates/site-api/src/prism_enrich.rs +++ b/crates/site-api/src/prism_enrich.rs @@ -11,56 +11,88 @@ use site_types::{ use site_data::map::{prism_submission, prism_telemetry}; use site_types::LeaderboardRow; -/// HuggingFace model card for the GPT-2 Large reference weights. -pub const GPT2_SOURCE_URL: &str = "https://huggingface.co/gpt2-large"; +/// HuggingFace model card for GPT-2 Large reference weights. +pub const GPT2_LARGE_SOURCE_URL: &str = "https://huggingface.co/gpt2-large"; +/// HuggingFace model card for GPT-2 Small (`openai-community/gpt2`). +pub const GPT2_SMALL_SOURCE_URL: &str = "https://huggingface.co/openai-community/gpt2"; /// Public disclaimer for the GPT-2 Large Prism-protocol reference row. -pub const GPT2_DISCLAIMER: &str = "Public GPT-2 Large (774M) — Prism-protocol eval-only on 1×RTX 5090 (HF `gpt2-large` weights, public eval pack, `PRISM_TEST_EVAL_CAPS=0`). Not a miner train; G6 has no train probe curve; G8 µP width knob unsupported (floor 0)."; +pub const GPT2_LARGE_DISCLAIMER: &str = "Public GPT-2 Large (774M) — Prism-protocol eval-only on 1×RTX 5090 (HF `gpt2-large` weights, public eval pack, `PRISM_TEST_EVAL_CAPS=0`). Not a miner train; G6 has no train probe curve; G8 µP width knob unsupported (floor 0)."; + +/// Public disclaimer for the GPT-2 Small Prism-protocol reference row. +pub const GPT2_SMALL_DISCLAIMER: &str = "Public GPT-2 Small (124M) — Prism-protocol eval-only on 1×RTX 5090 (HF `openai-community/gpt2` / `gpt2` weights, public eval pack, `PRISM_TEST_EVAL_CAPS=0`). Not a miner train; G6 has no train probe curve; G8 µP width knob unsupported (floor 0)."; /// GPT-2 Large parameter count (millions) — measured `n_params` / 1e6. pub const GPT2_LARGE_PARAMS_M: f64 = 774.0; +/// GPT-2 Small parameter count (millions) — measured `n_params` / 1e6. +pub const GPT2_SMALL_PARAMS_M: f64 = 124.4; + +/// Measured Prism validation BPB — GPT-2 Large. +pub const GPT2_LARGE_BPB: f64 = 4.163_851_322_121_356_4; +/// Measured Prism validation BPB — GPT-2 Small. +pub const GPT2_SMALL_BPB: f64 = 4.759_478_148_923_918; + +/// GPT-2 Large G2 benches (Prism public pack, 1×RTX 5090). +pub const GPT2_LARGE_HELLASWAG: f64 = 0.395; +pub const GPT2_LARGE_ARC_EASY: f64 = 0.28; +pub const GPT2_LARGE_ARC_CHALLENGE: f64 = 0.28; +pub const GPT2_LARGE_PIQA: f64 = 0.69; +pub const GPT2_LARGE_WINOGRANDE: f64 = 0.545; +pub const GPT2_LARGE_BOOLQ: f64 = 0.64; +pub const GPT2_LARGE_LAMBADA: f64 = 0.985; +pub const GPT2_LARGE_OPENBOOKQA: f64 = 0.335; + +/// GPT-2 Small G2 benches (Prism public pack, 1×RTX 5090). +pub const GPT2_SMALL_HELLASWAG: f64 = 0.355; +pub const GPT2_SMALL_ARC_EASY: f64 = 0.245; +pub const GPT2_SMALL_ARC_CHALLENGE: f64 = 0.24; +pub const GPT2_SMALL_PIQA: f64 = 0.585; +pub const GPT2_SMALL_WINOGRANDE: f64 = 0.515; +pub const GPT2_SMALL_BOOLQ: f64 = 0.575; +pub const GPT2_SMALL_LAMBADA: f64 = 0.97; +pub const GPT2_SMALL_OPENBOOKQA: f64 = 0.32; -/// Measured Prism validation BPB (frozen FineWeb-edu val cut, gpt2 tokenizer). -pub const GPT2_BPB: f64 = 4.163_851_322_121_356_4; - -/// HellaSwag `org.g2.hellaswag_acc` (Prism public pack). -pub const GPT2_HELLASWAG: f64 = 0.395; -/// ARC-Easy `org.g2.arc_easy_acc`. -pub const GPT2_ARC_EASY: f64 = 0.28; -/// ARC-Challenge `org.g2.arc_challenge_acc`. -pub const GPT2_ARC_CHALLENGE: f64 = 0.28; -/// PIQA `org.g2.piqa_acc`. -pub const GPT2_PIQA: f64 = 0.69; -/// WinoGrande `org.g2.winogrande_acc`. -pub const GPT2_WINOGRANDE: f64 = 0.545; -/// BoolQ `org.g2.boolq_acc`. -pub const GPT2_BOOLQ: f64 = 0.64; -/// LAMBADA `org.g2.lambada_acc` (Prism public pack on 1×RTX 5090). -pub const GPT2_LAMBADA: f64 = 0.985; -/// OpenBookQA `org.g2.obqa_acc` (Prism public pack on 1×RTX 5090). -pub const GPT2_OPENBOOKQA: f64 = 0.335; - -/// Frozen public GPT-2 baseline(s) for `GET …/references`. +/// Frozen public GPT-2 baselines for `GET …/references` (Large then Small). #[must_use] pub fn prism_reference_baselines() -> Vec { - vec![PrismReferenceBaseline { - id: "gpt2-large-774m".into(), - label: "Public GPT-2 Large (774M)".into(), - params_m: GPT2_LARGE_PARAMS_M, - bpb: Some(GPT2_BPB), - benchmarks: PrismBenchmarks { - hellaswag: Some(GPT2_HELLASWAG), - arc_easy: Some(GPT2_ARC_EASY), - arc_challenge: Some(GPT2_ARC_CHALLENGE), - piqa: Some(GPT2_PIQA), - winogrande: Some(GPT2_WINOGRANDE), - boolq: Some(GPT2_BOOLQ), - lambada: Some(GPT2_LAMBADA), - openbookqa: Some(GPT2_OPENBOOKQA), + vec![ + PrismReferenceBaseline { + id: "gpt2-large-774m".into(), + label: "Public GPT-2 Large (774M)".into(), + params_m: GPT2_LARGE_PARAMS_M, + bpb: Some(GPT2_LARGE_BPB), + benchmarks: PrismBenchmarks { + hellaswag: Some(GPT2_LARGE_HELLASWAG), + arc_easy: Some(GPT2_LARGE_ARC_EASY), + arc_challenge: Some(GPT2_LARGE_ARC_CHALLENGE), + piqa: Some(GPT2_LARGE_PIQA), + winogrande: Some(GPT2_LARGE_WINOGRANDE), + boolq: Some(GPT2_LARGE_BOOLQ), + lambada: Some(GPT2_LARGE_LAMBADA), + openbookqa: Some(GPT2_LARGE_OPENBOOKQA), + }, + source_url: GPT2_LARGE_SOURCE_URL.into(), + disclaimer: GPT2_LARGE_DISCLAIMER.into(), + }, + PrismReferenceBaseline { + id: "gpt2-small-124m".into(), + label: "Public GPT-2 Small (124M)".into(), + params_m: GPT2_SMALL_PARAMS_M, + bpb: Some(GPT2_SMALL_BPB), + benchmarks: PrismBenchmarks { + hellaswag: Some(GPT2_SMALL_HELLASWAG), + arc_easy: Some(GPT2_SMALL_ARC_EASY), + arc_challenge: Some(GPT2_SMALL_ARC_CHALLENGE), + piqa: Some(GPT2_SMALL_PIQA), + winogrande: Some(GPT2_SMALL_WINOGRANDE), + boolq: Some(GPT2_SMALL_BOOLQ), + lambada: Some(GPT2_SMALL_LAMBADA), + openbookqa: Some(GPT2_SMALL_OPENBOOKQA), + }, + source_url: GPT2_SMALL_SOURCE_URL.into(), + disclaimer: GPT2_SMALL_DISCLAIMER.into(), }, - source_url: GPT2_SOURCE_URL.into(), - disclaimer: GPT2_DISCLAIMER.into(), - }] + ] } /// Infer AutoModel vs legacy from a detail or list-shaped payload. @@ -516,18 +548,25 @@ mod tests { } #[test] - fn gpt2_baseline_is_prism_protocol_large() { + fn gpt2_baselines_are_prism_protocol_large_and_small() { let refs = prism_reference_baselines(); - assert_eq!(refs.len(), 1); + assert_eq!(refs.len(), 2); assert_eq!(refs[0].id, "gpt2-large-774m"); + assert_eq!(refs[1].id, "gpt2-small-124m"); assert!(refs[0].bpb.is_some()); - assert!((refs[0].bpb.unwrap() - GPT2_BPB).abs() < 1e-9); + assert!(refs[1].bpb.is_some()); + assert!((refs[0].bpb.unwrap() - GPT2_LARGE_BPB).abs() < 1e-9); + assert!((refs[1].bpb.unwrap() - GPT2_SMALL_BPB).abs() < 1e-9); assert!((refs[0].params_m - 774.0).abs() < f64::EPSILON); - assert!((refs[0].benchmarks.hellaswag.unwrap() - GPT2_HELLASWAG).abs() < f64::EPSILON); - assert!((refs[0].benchmarks.lambada.unwrap() - GPT2_LAMBADA).abs() < f64::EPSILON); - assert!((refs[0].benchmarks.openbookqa.unwrap() - GPT2_OPENBOOKQA).abs() < f64::EPSILON); + assert!((refs[1].params_m - 124.4).abs() < 1e-9); + assert!((refs[0].benchmarks.hellaswag.unwrap() - GPT2_LARGE_HELLASWAG).abs() < f64::EPSILON); + assert!((refs[1].benchmarks.hellaswag.unwrap() - GPT2_SMALL_HELLASWAG).abs() < f64::EPSILON); + assert!((refs[0].benchmarks.lambada.unwrap() - GPT2_LARGE_LAMBADA).abs() < f64::EPSILON); + assert!((refs[1].benchmarks.openbookqa.unwrap() - GPT2_SMALL_OPENBOOKQA).abs() < f64::EPSILON); assert!(refs[0].disclaimer.contains("Prism-protocol")); + assert!(refs[1].disclaimer.contains("Prism-protocol")); assert!(refs[0].source_url.contains("gpt2-large")); + assert!(refs[1].source_url.contains("openai-community/gpt2")); } #[test] diff --git a/docs/SITE_API.md b/docs/SITE_API.md index 62c84adbb..25dbaa911 100644 --- a/docs/SITE_API.md +++ b/docs/SITE_API.md @@ -59,10 +59,10 @@ Additional Prism routes: | Path | Response | |------|----------| | `GET /v1/site/arenas/prism/submissions/{id}` | `PrismSubmissionDetail` — list fields + `eval` summary (status, groups, gates, composite) + telemetry + public `review` / `similarity` (quality/kind only). **No** raw patch text. | -| `GET /v1/site/arenas/prism/references` | `PrismReferenceBaseline[]` — frozen **Prism-protocol** GPT-2 Large (774M) reference: measured val **`bpb`** + G2 benches (`hellaswag` / `arcEasy` / …) from a 1×RTX 5090 eval-only run on the public pack + HF `gpt2-large` weights. Includes `sourceUrl` / `disclaimer`. | +| `GET /v1/site/arenas/prism/references` | `PrismReferenceBaseline[]` — frozen **Prism-protocol** GPT-2 references (Large 774M **and** Small 124M): measured val **`bpb`** + G2 benches from 1×RTX 5090 eval-only runs on the public pack (`gpt2-large` + `openai-community/gpt2`). Includes `sourceUrl` / `disclaimer`. | | `GET /v1/site/arenas/prism/submissions/{id}/telemetry` | Existing loss-curve payload (also embedded on detail). | -GPT-2 Large constants live in `crates/site-api` (`prism_enrich`) so API and FE stay aligned; they are **measured Prism-protocol** numbers (eval-only, public pack), not Eleuther literature tables. List/leaderboard row shells still map in `crates/site-data`. +GPT-2 Large + Small constants live in `crates/site-api` (`prism_enrich`) so API and FE stay aligned; they are **measured Prism-protocol** numbers (eval-only, public pack), not Eleuther literature tables. List/leaderboard row shells still map in `crates/site-data`. `GET /v1/site/arenas/{slug}/submissions` and `/leaderboard` accept optional `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle, From f900e49013bd8459c1670681e2c7f10042b4ba73 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:01:51 +0000 Subject: [PATCH 2/3] style(site): rustfmt gpt2-small dual reference tables --- crates/prism-registry/src/hf.rs | 4 ---- crates/site-api/src/prism_enrich.rs | 12 +++++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/prism-registry/src/hf.rs b/crates/prism-registry/src/hf.rs index 79e7c70c3..7828b407f 100644 --- a/crates/prism-registry/src/hf.rs +++ b/crates/prism-registry/src/hf.rs @@ -60,7 +60,6 @@ const GPT2_SMALL_LAMBADA: f64 = 0.97; const GPT2_SMALL_OPENBOOKQA: f64 = 0.32; const GPT2_SMALL_SOURCE: &str = "https://huggingface.co/openai-community/gpt2"; - /// HuggingFace Hub publisher (token never `Debug`/`Display`'d). pub struct HfTopModelPublisher { http: reqwest::Client, @@ -772,8 +771,6 @@ fn train_tokens(metrics: Option<&serde_json::Value>) -> Option { .or_else(|| metric_u64(metrics, &["tokens_seen"]).filter(|&t| t > 10_000)) } - - fn verdict_higher(delta: f64) -> (&'static str, &'static str) { if delta.abs() < 1e-9 { ("=", "tie") @@ -840,7 +837,6 @@ fn bench_row_lower_dual( } } - const CONFIGURATION_PRISM_PY: &str = r#" """Prism custom-arch Hub config (trust_remote_code).""" diff --git a/crates/site-api/src/prism_enrich.rs b/crates/site-api/src/prism_enrich.rs index 96ce20aac..0334b72fd 100644 --- a/crates/site-api/src/prism_enrich.rs +++ b/crates/site-api/src/prism_enrich.rs @@ -559,10 +559,16 @@ mod tests { assert!((refs[1].bpb.unwrap() - GPT2_SMALL_BPB).abs() < 1e-9); assert!((refs[0].params_m - 774.0).abs() < f64::EPSILON); assert!((refs[1].params_m - 124.4).abs() < 1e-9); - assert!((refs[0].benchmarks.hellaswag.unwrap() - GPT2_LARGE_HELLASWAG).abs() < f64::EPSILON); - assert!((refs[1].benchmarks.hellaswag.unwrap() - GPT2_SMALL_HELLASWAG).abs() < f64::EPSILON); + assert!( + (refs[0].benchmarks.hellaswag.unwrap() - GPT2_LARGE_HELLASWAG).abs() < f64::EPSILON + ); + assert!( + (refs[1].benchmarks.hellaswag.unwrap() - GPT2_SMALL_HELLASWAG).abs() < f64::EPSILON + ); assert!((refs[0].benchmarks.lambada.unwrap() - GPT2_LARGE_LAMBADA).abs() < f64::EPSILON); - assert!((refs[1].benchmarks.openbookqa.unwrap() - GPT2_SMALL_OPENBOOKQA).abs() < f64::EPSILON); + assert!( + (refs[1].benchmarks.openbookqa.unwrap() - GPT2_SMALL_OPENBOOKQA).abs() < f64::EPSILON + ); assert!(refs[0].disclaimer.contains("Prism-protocol")); assert!(refs[1].disclaimer.contains("Prism-protocol")); assert!(refs[0].source_url.contains("gpt2-large")); From 43e549d34d30862ea15b1e3b11bbecaa22d505e6 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:11:26 +0000 Subject: [PATCH 3/3] fix(prism-registry): include_str hub py templates for loc-cap --- .../prism-registry/src/configuration_prism.py | 12 ++ crates/prism-registry/src/hf.rs | 110 +----------------- crates/prism-registry/src/modeling_prism.py | 92 +++++++++++++++ 3 files changed, 106 insertions(+), 108 deletions(-) create mode 100644 crates/prism-registry/src/configuration_prism.py create mode 100644 crates/prism-registry/src/modeling_prism.py diff --git a/crates/prism-registry/src/configuration_prism.py b/crates/prism-registry/src/configuration_prism.py new file mode 100644 index 000000000..30c5f81ca --- /dev/null +++ b/crates/prism-registry/src/configuration_prism.py @@ -0,0 +1,12 @@ +"""Prism custom-arch Hub config (trust_remote_code).""" + +from transformers import PretrainedConfig + + +class PrismConfig(PretrainedConfig): + model_type = "prism_custom" + + def __init__(self, prism_arch_id=None, checkpoint_file="checkpoint.pt", **kwargs): + super().__init__(**kwargs) + self.prism_arch_id = prism_arch_id + self.checkpoint_file = checkpoint_file diff --git a/crates/prism-registry/src/hf.rs b/crates/prism-registry/src/hf.rs index 7828b407f..e3b290bb7 100644 --- a/crates/prism-registry/src/hf.rs +++ b/crates/prism-registry/src/hf.rs @@ -837,115 +837,9 @@ fn bench_row_lower_dual( } } -const CONFIGURATION_PRISM_PY: &str = r#" -"""Prism custom-arch Hub config (trust_remote_code).""" +const CONFIGURATION_PRISM_PY: &str = include_str!("configuration_prism.py"); -from transformers import PretrainedConfig - - -class PrismConfig(PretrainedConfig): - model_type = "prism_custom" - - def __init__(self, prism_arch_id=None, checkpoint_file="checkpoint.pt", **kwargs): - super().__init__(**kwargs) - self.prism_arch_id = prism_arch_id - self.checkpoint_file = checkpoint_file -"#; - -const MODELING_PRISM_PY: &str = r#" -"""Prism custom-arch loader (trust_remote_code). - -Supports: - 1. Legacy seam: ``architecture.build_model(ctx)`` + ``checkpoint.pt`` - 2. AutoModel novelty: sources under ``sources/`` (apply patch offline) - -This is intentionally permissive — novel arches are not limited to stock -GPT-2 ``transformers`` configs. -""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path -from typing import Any, Optional - -import torch -from torch import nn -from transformers import PreTrainedModel - -try: - from configuration_prism import PrismConfig -except ImportError: # package-style local import - from .configuration_prism import PrismConfig - - -def _load_architecture_module(root: Path): - path = root / "architecture.py" - if not path.is_file(): - raise FileNotFoundError(f"architecture.py missing under {root}") - spec = importlib.util.spec_from_file_location("prism_architecture", path) - if spec is None or spec.loader is None: - raise ImportError("cannot load architecture.py") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -class PrismCustomModel(PreTrainedModel): - config_class = PrismConfig - _no_split_modules = [] - - def __init__(self, config: PrismConfig, inner: Optional[nn.Module] = None): - super().__init__(config) - self.inner = inner if inner is not None else nn.Identity() - self.post_init() - - def forward(self, *args: Any, **kwargs: Any): - return self.inner(*args, **kwargs) - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): - trust = kwargs.pop("trust_remote_code", True) - config = kwargs.pop("config", None) - if config is None: - config = PrismConfig.from_pretrained( - pretrained_model_name_or_path, trust_remote_code=trust, **kwargs - ) - root = Path(pretrained_model_name_or_path) - # Hub download may leave us with a cache dir; prefer local folder layout. - if not (root / "architecture.py").is_file(): - # Fall back to empty shell when only config is present. - return cls(config) - - mod = _load_architecture_module(root) - ctx = { - "device": "cpu", - "dtype": torch.float32, - "seed": 0, - "vocab_size": getattr(config, "vocab_size", 50257), - } - if hasattr(mod, "build_model"): - inner = mod.build_model(ctx) - elif hasattr(mod, "Model"): - inner = mod.Model(**{k: v for k, v in ctx.items() if k in ("vocab_size",)}) - else: - raise AttributeError( - "architecture.py must define build_model(ctx) or Model for Hub reload" - ) - model = cls(config, inner=inner) - ckpt_name = getattr(config, "checkpoint_file", "checkpoint.pt") or "checkpoint.pt" - ckpt = root / ckpt_name - if ckpt.is_file(): - blob = torch.load(ckpt, map_location="cpu", weights_only=False) - state = blob.get("state_dict", blob) if isinstance(blob, dict) else blob - if isinstance(state, dict): - try: - model.inner.load_state_dict(state, strict=False) - except Exception: - # Novel arches / tied keys — best-effort; sources remain authoritative. - pass - return model -"#; +const MODELING_PRISM_PY: &str = include_str!("modeling_prism.py"); /// Collect novel source files from a packed `tree_blob` for Hub `sources/`. #[must_use] diff --git a/crates/prism-registry/src/modeling_prism.py b/crates/prism-registry/src/modeling_prism.py new file mode 100644 index 000000000..e7b408dc3 --- /dev/null +++ b/crates/prism-registry/src/modeling_prism.py @@ -0,0 +1,92 @@ +"""Prism custom-arch loader (trust_remote_code). + +Supports: + 1. Legacy seam: ``architecture.build_model(ctx)`` + ``checkpoint.pt`` + 2. AutoModel novelty: sources under ``sources/`` (apply patch offline) + +This is intentionally permissive — novel arches are not limited to stock +GPT-2 ``transformers`` configs. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any, Optional + +import torch +from torch import nn +from transformers import PreTrainedModel + +try: + from configuration_prism import PrismConfig +except ImportError: # package-style local import + from .configuration_prism import PrismConfig + + +def _load_architecture_module(root: Path): + path = root / "architecture.py" + if not path.is_file(): + raise FileNotFoundError(f"architecture.py missing under {root}") + spec = importlib.util.spec_from_file_location("prism_architecture", path) + if spec is None or spec.loader is None: + raise ImportError("cannot load architecture.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class PrismCustomModel(PreTrainedModel): + config_class = PrismConfig + _no_split_modules = [] + + def __init__(self, config: PrismConfig, inner: Optional[nn.Module] = None): + super().__init__(config) + self.inner = inner if inner is not None else nn.Identity() + self.post_init() + + def forward(self, *args: Any, **kwargs: Any): + return self.inner(*args, **kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + trust = kwargs.pop("trust_remote_code", True) + config = kwargs.pop("config", None) + if config is None: + config = PrismConfig.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=trust, **kwargs + ) + root = Path(pretrained_model_name_or_path) + # Hub download may leave us with a cache dir; prefer local folder layout. + if not (root / "architecture.py").is_file(): + # Fall back to empty shell when only config is present. + return cls(config) + + mod = _load_architecture_module(root) + ctx = { + "device": "cpu", + "dtype": torch.float32, + "seed": 0, + "vocab_size": getattr(config, "vocab_size", 50257), + } + if hasattr(mod, "build_model"): + inner = mod.build_model(ctx) + elif hasattr(mod, "Model"): + inner = mod.Model(**{k: v for k, v in ctx.items() if k in ("vocab_size",)}) + else: + raise AttributeError( + "architecture.py must define build_model(ctx) or Model for Hub reload" + ) + model = cls(config, inner=inner) + ckpt_name = getattr(config, "checkpoint_file", "checkpoint.pt") or "checkpoint.pt" + ckpt = root / ckpt_name + if ckpt.is_file(): + blob = torch.load(ckpt, map_location="cpu", weights_only=False) + state = blob.get("state_dict", blob) if isinstance(blob, dict) else blob + if isinstance(state, dict): + try: + model.inner.load_state_dict(state, strict=False) + except Exception: + # Novel arches / tied keys — best-effort; sources remain authoritative. + pass + return model