diff --git a/bins/prism-challenge/src/main.rs b/bins/prism-challenge/src/main.rs index 08cf6e24a..3edeb5cfa 100644 --- a/bins/prism-challenge/src/main.rs +++ b/bins/prism-challenge/src/main.rs @@ -131,6 +131,13 @@ enum Cmd { #[arg(long, default_value_t = 500)] limit: u32, }, + /// Force-publish one submission to GitHub/HF top-model (operator republish). + /// Requires DB + `PRISM_TOPMODEL_*` token files + parked checkpoint when + /// weights are required. + RepublishTopmodel { + /// Submission id (64 hex). + id: String, + }, } fn main() -> ExitCode { @@ -163,6 +170,13 @@ fn run(cli: Cli) -> Result<(), String> { .map_err(|e| e.to_string())?; return rt.block_on(cmd_rescore_g2(*dry_run, id.clone(), *limit)); } + Some(Cmd::RepublishTopmodel { id }) => { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + return rt.block_on(cmd_republish_topmodel(id.clone())); + } _ => {} } let rt = tokio::runtime::Builder::new_multi_thread() @@ -265,6 +279,44 @@ async fn cmd_rescore_g2(dry_run: bool, id: Option, limit: u32) -> Result Ok(()) } +async fn cmd_republish_topmodel(id: String) -> Result<(), String> { + let url = std::env::var("BASE_DATABASE_URL") + .map_err(|_| "BASE_DATABASE_URL required for republish-topmodel".to_string())?; + let pool = db::connect(&url).await.map_err(|e| e.to_string())?; + let store: Arc = Arc::new(DbPrismStore::new(pool)); + let row = store + .get(&id) + .await + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("unknown submission {id}"))?; + let lattice_score = match &row.final_score { + Some(FinalScore::Score(v)) if *v > 0 => *v, + _ => return Err(format!("submission {id} has no positive lattice score")), + }; + if !row.weight_eligible() { + return Err(format!( + "submission {id} is not weight-eligible (AutoModel 2.0)" + )); + } + let gh = build_topmodel(); + println!( + "republish-topmodel id={id} score={lattice_score} bpb={:?} arch={:?} github={}", + row.bpb, + row.arch_id, + gh.is_some() + ); + prism_registry::post_score_hooks(&store, gh.as_deref(), &row, true).await; + if let Ok(Some(pub_row)) = store.last_publication().await { + println!( + "publication submission_id={} repo={} commit={:?}", + pub_row.submission_id, pub_row.repo_path, pub_row.commit_sha + ); + } else { + println!("publication: no journal row (publish may have failed; check logs)"); + } + Ok(()) +} + /// Resolve the Lium API key (file/env/credentials — never logged). fn load_lium_api_key() -> Option { if let Ok(path) = std::env::var("LIUM_API_KEY_FILE") { diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index 3d0a640b2..ba45359c9 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -481,7 +481,8 @@ impl Orchestrator { .map_err(|e| e.to_string())?; if let Ok(Some(scored)) = self.store.get(&id).await { - prism_registry::post_score_hooks(&self.store, self.topmodel.as_deref(), &scored).await; + prism_registry::post_score_hooks(&self.store, self.topmodel.as_deref(), &scored, false) + .await; } self.logs.clear(&id); Ok(()) @@ -988,12 +989,8 @@ impl Orchestrator { .await .map_err(|e| e.to_string())?; if let Some(s) = &summary { - info!( - epoch = s.epoch, - leaves = s.leaves, - batch = s.batch, - "epoch leaf set emitted" - ); + #[rustfmt::skip] + info!(epoch = s.epoch, leaves = s.leaves, batch = s.batch, "epoch leaf set emitted"); } Ok(summary) } diff --git a/crates/prism-challenge/tests/arch_competition.rs b/crates/prism-challenge/tests/arch_competition.rs index 985e24e8e..faf534403 100644 --- a/crates/prism-challenge/tests/arch_competition.rs +++ b/crates/prism-challenge/tests/arch_competition.rs @@ -349,7 +349,7 @@ async fn topmodel_hooks_graceful_without_publisher() { r.metrics_json = Some(serde_json::json!({"n_params": 12_000_000})); store.insert_queued(&r).await.unwrap(); - prism_registry::post_score_hooks(&store, None, &r).await; + prism_registry::post_score_hooks(&store, None, &r, false).await; // Arch published even without a GitHub publisher; nothing journaled. assert!(store diff --git a/crates/prism-registry/src/hf.rs b/crates/prism-registry/src/hf.rs index e3b290bb7..8494b6946 100644 --- a/crates/prism-registry/src/hf.rs +++ b/crates/prism-registry/src/hf.rs @@ -1,7 +1,8 @@ //! HuggingFace Hub top-model publisher. //! -//! When a submission becomes the new global-best bpb, the master publishes a -//! **reloadable** Hub model card to `PRISM_TOPMODEL_HF_REPO` (default +//! When a submission becomes the new global-best **lattice score** (G2 board +//! ranking — never min-bpb alone), the master publishes a **reloadable** Hub +//! model card to `PRISM_TOPMODEL_HF_REPO` (default //! `BaseIntelligence/top-prism-architecture`): //! //! - custom architecture / AutoModel novelty sources (`architecture.py`, diff --git a/crates/prism-registry/src/hooks.rs b/crates/prism-registry/src/hooks.rs index 733f3a568..b955b353e 100644 --- a/crates/prism-registry/src/hooks.rs +++ b/crates/prism-registry/src/hooks.rs @@ -8,10 +8,12 @@ //! their arch. Idempotent on digest (simultaneous duplicates share the //! first registration). //! 2. **Arch best bpb** — lower-wins update feeding owner credit + the -//! public leaderboard (any trainer's result counts). -//! 3. **Top-model publish** — when the row's bpb is a new global best -//! (≤ best scored bpb ever AND < last published bpb), publish to GitHub -//! and journal the publication. No-op without a configured publisher. +//! public leaderboard (any trainer's result counts; audit / secondary). +//! 3. **Top-model publish** — when the row's **lattice score** is a new +//! global best (≥ best scored score ever AND > last published score), +//! publish to GitHub/HF and journal. Matches live board ranking +//! (G2 benchmark lattice under `scoring_version` 4). Never min-bpb alone. +//! No-op without a configured publisher. use std::sync::Arc; @@ -24,16 +26,20 @@ use tracing::{info, warn}; use crate::publish::{TopModelPublisher, TopModelRequest, TOPMODEL_REPO_PATH}; /// Run registry + top-model bookkeeping for one finalized row. +/// +/// When `force` is true (operator republish CLI), skip the global-best / +/// beats-published guards and publish anyway. #[allow(clippy::too_many_lines)] pub async fn post_score_hooks( store: &Arc, publisher: Option<&TopModelPublisher>, row: &SubmissionState, + force: bool, ) { - let (Some(bpb), Some(FinalScore::Score(v))) = (row.bpb, &row.final_score) else { + let (Some(bpb), Some(FinalScore::Score(lattice))) = (row.bpb, &row.final_score) else { return; }; - if *v == 0 { + if *lattice == 0 { return; // cheat / copy-gate zero never publishes nor sets arch best } @@ -83,9 +89,9 @@ pub async fn post_score_hooks( } } - // (3) Top-model publish on a new global best — GitHub (optional) + HF - // (optional). Both require a verified secure-receive receipt when - // `PRISM_TOPMODEL_REQUIRE_WEIGHTS=1` (default); source-only is opt-in. + // (3) Top-model publish on a new global-best **lattice score** — GitHub + // (optional) + HF (optional). Both require a verified secure-receive + // receipt when `PRISM_TOPMODEL_REQUIRE_WEIGHTS=1` (default). // Recipe 2.0 / AutoModel only — legacy 1.x never becomes the published // top-model champion (historical FE rows stay in Postgres). if !row.weight_eligible() { @@ -95,12 +101,14 @@ pub async fn post_score_hooks( ); return; } - let last = store.last_publication_bpb().await.unwrap_or(None); - let global = store.best_scored_bpb().await.unwrap_or(None); - let is_global_best = global.is_some_and(|g| bpb <= g); - let beats_published = last.is_none_or(|l| bpb < l); - if !(is_global_best && beats_published) { - return; + if force { + info!(submission_id = %row.id, score = lattice, "top-model: force republish"); + } else { + let last = store.last_publication_score().await.unwrap_or(None); + let global = store.best_scored_score().await.unwrap_or(None); + if !(global.is_some_and(|g| *lattice >= g) && last.is_none_or(|l| *lattice > l)) { + return; + } } let ckpt = match prism_artifacts::verify_parked(&row.id) { Ok(receipt) => { @@ -139,7 +147,7 @@ pub async fn post_score_hooks( if let Some(publisher) = publisher { match publisher.publish(&req).await { Ok(sha) => { - info!(submission_id = %row.id, bpb, commit = %sha, "top model published to GitHub"); + info!(submission_id = %row.id, score = lattice, bpb, commit = %sha, "top model published"); let rec = TopModelPublication { submission_id: row.id.clone(), arch_id: arch_id.clone(), diff --git a/crates/prism-registry/src/lib.rs b/crates/prism-registry/src/lib.rs index a49734ef8..1bd1b798b 100644 --- a/crates/prism-registry/src/lib.rs +++ b/crates/prism-registry/src/lib.rs @@ -5,8 +5,9 @@ //! - [`competition_scores`] — per-epoch emission math for the architecture //! competition (SCORE_MAX lattice preserved; exact rule documented in //! `docs/PRISM.md` § Architecture competition). -//! - [`TopModelPublisher`] — publishes each new global-best bpb model to the -//! public `BaseIntelligence/prism` GitHub repo under `top-model/`, via a +//! - [`TopModelPublisher`] — publishes each new global-best **lattice score** +//! model (G2 benches under `scoring_version` 4 — never min-bpb alone) to +//! the public `BaseIntelligence/prism` GitHub repo under `top-model/`, via a //! token read from a deploy secret file (`PRISM_TOPMODEL_GITHUB_TOKEN_FILE`; //! graceful no-op when absent). //! - [`HfTopModelPublisher`] — same trigger, commits a reloadable custom-arch diff --git a/crates/prism-registry/src/publish.rs b/crates/prism-registry/src/publish.rs index 485ae9f9c..74823c5cd 100644 --- a/crates/prism-registry/src/publish.rs +++ b/crates/prism-registry/src/publish.rs @@ -1,7 +1,7 @@ -//! Top-model GitHub publisher: each new global-best bpb model is published -//! to the public `BaseIntelligence/prism` repo under `top-model/` -//! (architecture.py + training.py + METRICS.json + README.md block) via the -//! GitHub contents API. +//! Top-model GitHub publisher: each new global-best **lattice score** model +//! (G2 benchmark board ranking) is published to the public +//! `BaseIntelligence/prism` repo under `top-model/` (architecture.py + +//! training.py + METRICS.json + README.md block) via the GitHub contents API. //! //! Token discipline: the GitHub token is read from a deploy secret **file** //! (`PRISM_TOPMODEL_GITHUB_TOKEN_FILE`, e.g. `deploy/secrets/github/token`), @@ -51,7 +51,7 @@ pub struct TopModelRequest { pub arch_id: Option, /// Miner hotkey that set the best. pub owner_hotkey: String, - /// Global-best bpb. + /// Measured bpb (audit / README; champion selection uses lattice score). pub bpb: f64, /// architecture.py (registry source for training-only entries). pub architecture_py: String, @@ -240,8 +240,9 @@ fn readme_block(req: &TopModelRequest, weight_note: &str) -> String { }; format!( "# PRISM top model\n\n\ - Published by the Base master on every new global-best bpb. This\n\ - directory always mirrors the current champion; history lives in git.\n\n\ + Published by the Base master on every new global-best G2 lattice\n\ + score (live board ranking). This directory always mirrors the\n\ + current champion; history lives in git.\n\n\ | field | value |\n|---|---|\n\ | arch_id | `{}` |\n\ | owner_hotkey | `{}…` |\n\ diff --git a/crates/prism-store/src/arch.rs b/crates/prism-store/src/arch.rs index 6c746983f..6a185271b 100644 --- a/crates/prism-store/src/arch.rs +++ b/crates/prism-store/src/arch.rs @@ -197,6 +197,19 @@ pub(crate) async fn last_publication_bpb(pool: &PgPool) -> Result, S Ok(last_publication(pool).await?.map(|p| p.bpb)) } +pub(crate) async fn last_publication_score(pool: &PgPool) -> Result, StoreError> { + let row: Option<(Option,)> = sqlx::query_as( + "SELECT s.score FROM prism_topmodel_publication p \ + JOIN prism_submission s ON s.id = p.submission_id \ + WHERE s.kind = 'score' AND s.score > 0 \ + ORDER BY p.published_at DESC LIMIT 1", + ) + .fetch_optional(pool) + .await + .map_err(backend)?; + Ok(row.and_then(|(s,)| s.map(i64::cast_unsigned))) +} + pub(crate) async fn last_publication( pool: &PgPool, ) -> Result, StoreError> { @@ -232,3 +245,16 @@ pub(crate) async fn best_scored_bpb(pool: &PgPool) -> Result, StoreE .map_err(backend)?; Ok(row.0) } + +pub(crate) async fn best_scored_score(pool: &PgPool) -> Result, StoreError> { + let row: (Option,) = sqlx::query_as(&format!( + "SELECT MAX(score) FROM prism_submission \ + WHERE kind = 'score' AND score > 0 \ + AND {}", + crate::emit::WEIGHT_ELIGIBLE_SQL + )) + .fetch_one(pool) + .await + .map_err(backend)?; + Ok(row.0.map(i64::cast_unsigned)) +} diff --git a/crates/prism-store/src/dbprism.rs b/crates/prism-store/src/dbprism.rs index e03ab4297..9bb1e71cd 100644 --- a/crates/prism-store/src/dbprism.rs +++ b/crates/prism-store/src/dbprism.rs @@ -426,6 +426,10 @@ impl PrismStore for DbPrismStore { arch::last_publication_bpb(&self.pool).await } + async fn last_publication_score(&self) -> Result, StoreError> { + arch::last_publication_score(&self.pool).await + } + async fn last_publication(&self) -> Result, StoreError> { arch::last_publication(&self.pool).await } @@ -434,6 +438,10 @@ impl PrismStore for DbPrismStore { arch::best_scored_bpb(&self.pool).await } + async fn best_scored_score(&self) -> Result, StoreError> { + arch::best_scored_score(&self.pool).await + } + async fn list_stuck(&self, grace_secs: u64) -> Result, StoreError> { let rows = dbs::stuck_prism_before_grace( &self.pool, diff --git a/crates/prism-store/src/store.rs b/crates/prism-store/src/store.rs index b2007cc13..055803798 100644 --- a/crates/prism-store/src/store.rs +++ b/crates/prism-store/src/store.rs @@ -118,17 +118,25 @@ pub trait PrismStore: Send + Sync + std::fmt::Debug { /// Journal one top-model publication. async fn record_publication(&self, p: &TopModelPublication) -> Result<(), StoreError>; - /// bpb of the most recent publication (idempotency guard; `None` = never - /// published). + /// bpb of the most recent publication (audit; prefer [`Self::last_publication_score`]). async fn last_publication_bpb(&self) -> Result, StoreError>; + /// Lattice score of the submission behind the most recent publication + /// (idempotency guard for G2 / score ranking; `None` = never published or + /// the published row has no positive score). + async fn last_publication_score(&self) -> Result, StoreError>; + /// Most recent top-model publication row (`None` = never published). async fn last_publication(&self) -> Result, StoreError>; - /// Best (lowest) bpb across all scored submissions ever (global top-model - /// trigger baseline). + /// Best (lowest) bpb across weight-eligible scored submissions (audit / + /// legacy). Top-model publish uses [`Self::best_scored_score`]. async fn best_scored_bpb(&self) -> Result, StoreError>; + /// Best (highest) lattice score across weight-eligible scored submissions + /// (global top-model / HF champion trigger — matches live board ranking). + async fn best_scored_score(&self) -> Result, StoreError>; + /// Non-terminal rows beyond grace — for the stuck sweep. async fn list_stuck(&self, grace_secs: u64) -> Result, StoreError>; @@ -174,6 +182,10 @@ fn now_ms() -> u64 { .map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) } +fn lock(m: &Mutex) -> Result, StoreError> { + m.lock().map_err(|_| StoreError::Backend("poison".into())) +} + /// Extract the miner-reported loss series from a `RemoteExecResult` JSON blob /// (`telemetry.loss_series`). Empty when absent or malformed — the harness /// controls the shape, so a parse miss means a pre-telemetry recipe. @@ -188,10 +200,7 @@ pub(crate) fn telemetry_from_metrics(metrics_json: &serde_json::Value) -> Vec Result<(), StoreError> { - let mut rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut rows = lock(&self.rows)?; // Match Postgres unique(id): reject duplicates so a second POST cannot // enqueue another billable Lium claim for the same submission_id. if rows.iter().any(|r| r.id == row.id) { @@ -202,20 +211,11 @@ impl PrismStore for MemoryPrismStore { } async fn get(&self, id: &str) -> Result, StoreError> { - Ok(self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .iter() - .find(|r| r.id == id) - .cloned()) + Ok(lock(&self.rows)?.iter().find(|r| r.id == id).cloned()) } async fn claim_next(&self) -> Result, StoreError> { - let mut rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut rows = lock(&self.rows)?; let pos = rows.iter().position(|r| r.status == Stage::Queued); let Some(i) = pos else { return Ok(None) }; let mut row = rows.remove(i).ok_or(StoreError::Backend("pop".into()))?; @@ -230,10 +230,7 @@ impl PrismStore for MemoryPrismStore { update: &StatePatch, event: Option<&StageEvent>, ) -> Result { - let mut rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut rows = lock(&self.rows)?; let row = rows .iter_mut() .find(|r| r.id == id) @@ -278,17 +275,11 @@ impl PrismStore for MemoryPrismStore { if let Some(m) = &update.metrics_json { let series = telemetry_from_metrics(m); if !series.is_empty() { - self.telemetry - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .insert(id.to_owned(), series); + lock(&self.telemetry)?.insert(id.to_owned(), series); } } if let Some(e) = event { - self.events - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .push((id.to_owned(), e.clone())); + lock(&self.events)?.push((id.to_owned(), e.clone())); } Ok(out) } @@ -298,10 +289,7 @@ impl PrismStore for MemoryPrismStore { id: &str, bump_retry: bool, ) -> Result { - let mut rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut rows = lock(&self.rows)?; let row = rows .iter_mut() .find(|r| r.id == id) @@ -329,15 +317,9 @@ impl PrismStore for MemoryPrismStore { let out = row.clone(); drop(rows); // A re-scored row must re-enter the emission outbox. - self.emitted - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .remove(id); + lock(&self.emitted)?.remove(id); if !retained_measurement { - self.telemetry - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .remove(id); + lock(&self.telemetry)?.remove(id); } Ok(out) } @@ -350,10 +332,7 @@ impl PrismStore for MemoryPrismStore { ) -> Result, StoreError> { let st = status.and_then(Stage::parse); let miner_norm = miner.map(|m| m.trim().to_ascii_lowercase()); - let mut v: Vec<_> = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? + let mut v: Vec<_> = lock(&self.rows)? .iter() .filter(|r| st.is_none() || Some(r.status) == st) .filter(|r| { @@ -369,10 +348,7 @@ impl PrismStore for MemoryPrismStore { } async fn list_champions(&self, limit: u32) -> Result, StoreError> { - let mut v: Vec<_> = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? + let mut v: Vec<_> = lock(&self.rows)? .iter() .filter(|r| matches!(r.final_score, Some(FinalScore::Score(s)) if s > 0)) .cloned() @@ -383,10 +359,7 @@ impl PrismStore for MemoryPrismStore { } async fn events(&self, id: &str) -> Result, StoreError> { - Ok(self - .events - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? + Ok(lock(&self.events)? .iter() .filter(|(k, _)| k == id) .map(|(_, e)| e.clone()) @@ -394,13 +367,7 @@ impl PrismStore for MemoryPrismStore { } async fn telemetry(&self, id: &str) -> Result, StoreError> { - Ok(self - .telemetry - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .get(id) - .cloned() - .unwrap_or_default()) + Ok(lock(&self.telemetry)?.get(id).cloned().unwrap_or_default()) } async fn assign_emit_batch( @@ -408,14 +375,8 @@ impl PrismStore for MemoryPrismStore { netuid: u16, epoch: u64, ) -> Result, StoreError> { - let rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; - let mut emitted = self - .emitted - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let rows = lock(&self.rows)?; + let mut emitted = lock(&self.emitted)?; let mut out = Vec::new(); for r in rows.iter() { if r.netuid == netuid && r.final_score.is_some() && !emitted.contains_key(&r.id) { @@ -432,14 +393,8 @@ impl PrismStore for MemoryPrismStore { } async fn emit_batch(&self, netuid: u16, epoch: u64) -> Result, StoreError> { - let rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; - let emitted = self - .emitted - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let rows = lock(&self.rows)?; + let emitted = lock(&self.emitted)?; Ok(rows .iter() .filter(|r| { @@ -455,10 +410,7 @@ impl PrismStore for MemoryPrismStore { } async fn active_score_rows(&self, netuid: u16) -> Result, StoreError> { - let rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let rows = lock(&self.rows)?; Ok(rows .iter() .filter(|r| r.netuid == netuid && r.weight_eligible()) @@ -475,19 +427,11 @@ impl PrismStore for MemoryPrismStore { } async fn emit_cursor(&self, netuid: u16) -> Result, StoreError> { - Ok(self - .cursors - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .get(&netuid) - .copied()) + Ok(lock(&self.cursors)?.get(&netuid).copied()) } async fn set_emit_cursor(&self, netuid: u16, epoch: u64) -> Result<(), StoreError> { - let mut cursors = self - .cursors - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut cursors = lock(&self.cursors)?; let e = cursors.entry(netuid).or_insert(0); *e = (*e).max(epoch); Ok(()) @@ -499,14 +443,8 @@ impl PrismStore for MemoryPrismStore { Some(c) => c.saturating_add(1), None => 0, }; - let rows = self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; - let emitted = self - .emitted - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let rows = lock(&self.rows)?; + let emitted = lock(&self.emitted)?; Ok(rows .iter() .filter(|r| r.netuid == netuid) @@ -521,10 +459,7 @@ impl PrismStore for MemoryPrismStore { &self, rec: &ArchitectureRecord, ) -> Result { - let mut archs = self - .archs - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut archs = lock(&self.archs)?; if let Some(existing) = archs.values().find(|a| a.arch_digest == rec.arch_digest) { return Ok(PublishArchOutcome::Duplicate(existing.arch_id.clone())); } @@ -533,32 +468,18 @@ impl PrismStore for MemoryPrismStore { } async fn get_arch(&self, arch_id: &str) -> Result, StoreError> { - Ok(self - .archs - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .get(arch_id) - .cloned()) + Ok(lock(&self.archs)?.get(arch_id).cloned()) } async fn list_archs(&self, limit: u32) -> Result, StoreError> { - let mut v: Vec<_> = self - .archs - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .values() - .cloned() - .collect(); + let mut v: Vec<_> = lock(&self.archs)?.values().cloned().collect(); v.sort_by_key(|r| std::cmp::Reverse(r.created_at_ms)); v.truncate(limit as usize); Ok(v) } async fn note_arch_best_bpb(&self, arch_id: &str, bpb: f64) -> Result { - let mut archs = self - .archs - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut archs = lock(&self.archs)?; let Some(rec) = archs.get_mut(arch_id) else { return Ok(false); }; @@ -570,46 +491,42 @@ impl PrismStore for MemoryPrismStore { } async fn arch_owners(&self) -> Result, StoreError> { - Ok(self - .archs - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? + Ok(lock(&self.archs)? .values() .map(|a| (a.arch_id.clone(), a.owner_hotkey.clone())) .collect()) } async fn record_publication(&self, p: &TopModelPublication) -> Result<(), StoreError> { - self.publications - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .push(p.clone()); + lock(&self.publications)?.push(p.clone()); Ok(()) } async fn last_publication_bpb(&self) -> Result, StoreError> { - Ok(self - .publications - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .last() - .map(|p| p.bpb)) + Ok(lock(&self.publications)?.last().map(|p| p.bpb)) + } + + async fn last_publication_score(&self) -> Result, StoreError> { + let last = self.last_publication().await?; + let Some(p) = last else { + return Ok(None); + }; + let rows = lock(&self.rows)?; + Ok(rows + .iter() + .find(|r| r.id == p.submission_id) + .and_then(|r| match r.final_score { + Some(FinalScore::Score(v)) if v > 0 => Some(v), + _ => None, + })) } async fn last_publication(&self) -> Result, StoreError> { - Ok(self - .publications - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? - .last() - .cloned()) + Ok(lock(&self.publications)?.last().cloned()) } async fn best_scored_bpb(&self) -> Result, StoreError> { - Ok(self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? + Ok(lock(&self.rows)? .iter() .filter(|r| r.weight_eligible()) .filter(|r| matches!(r.final_score, Some(FinalScore::Score(v)) if v > 0)) @@ -617,12 +534,20 @@ impl PrismStore for MemoryPrismStore { .min_by(f64::total_cmp)) } + async fn best_scored_score(&self) -> Result, StoreError> { + Ok(lock(&self.rows)? + .iter() + .filter(|r| r.weight_eligible()) + .filter_map(|r| match r.final_score { + Some(FinalScore::Score(v)) if v > 0 => Some(v), + _ => None, + }) + .max()) + } + async fn list_stuck(&self, grace_secs: u64) -> Result, StoreError> { let cutoff = now_ms().saturating_sub(grace_secs.saturating_mul(1000)); - Ok(self - .rows - .lock() - .map_err(|_| StoreError::Backend("poison".into()))? + Ok(lock(&self.rows)? .iter() .filter(|r| !r.status.is_terminal() && r.updated_at_ms < cutoff) .cloned() @@ -630,10 +555,7 @@ impl PrismStore for MemoryPrismStore { } async fn precheck_quota_get(&self, identity: &str, day: &str) -> Result { - let map = self - .precheck_quota - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let map = lock(&self.precheck_quota)?; Ok(map .get(&(identity.to_owned(), day.to_owned())) .copied() @@ -646,10 +568,7 @@ impl PrismStore for MemoryPrismStore { day: &str, limit: u32, ) -> Result, StoreError> { - let mut map = self - .precheck_quota - .lock() - .map_err(|_| StoreError::Backend("poison".into()))?; + let mut map = lock(&self.precheck_quota)?; let key = (identity.to_owned(), day.to_owned()); let used = map.get(&key).copied().unwrap_or(0); if used >= limit { diff --git a/crates/site-api/src/handlers.rs b/crates/site-api/src/handlers.rs index 3512fd3e1..7a24d4194 100644 --- a/crates/site-api/src/handlers.rs +++ b/crates/site-api/src/handlers.rs @@ -477,7 +477,7 @@ async fn prism_leaderboard_json( "total": page_out.total, "pageCount": page_out.page_count, "epoch": epoch, - "metric": "bpb", + "metric": "score", "updatedAt": now_iso(), }) } @@ -1260,8 +1260,9 @@ mod tests { let (s, v) = call(app.clone(), "/v1/site/arenas/prism/leaderboard").await; assert_eq!(s, StatusCode::OK, "{v}"); - assert_eq!(v["metric"], "bpb"); - assert_eq!(v["items"][0]["elo"], 1.25); + assert_eq!(v["metric"], "score"); + // Lattice score in `elo`; measured bpb stays secondary. + assert_eq!(v["items"][0]["elo"], 900.0); assert_eq!(v["items"][0]["bpb"], 1.25); assert_eq!(v["items"][0]["paramsM"], 12.0); diff --git a/crates/site-data/src/map.rs b/crates/site-data/src/map.rs index e9dd813c4..6c9d63c48 100644 --- a/crates/site-data/src/map.rs +++ b/crates/site-data/src/map.rs @@ -644,16 +644,18 @@ pub fn is_prism_champion_submission(row: &Value) -> bool { .unwrap_or(false) } -/// Prism BPB leaderboard from **champion** terminal submissions (Score>0). +/// Prism champion leaderboard from terminal submissions (`Score>0`). /// -/// Non-top submissions are hidden from the public board. `elo` carries the BPB -/// value so the existing leaderboard row contract can surface rankings without -/// inventing Elo/duels; `bpb` / `paramsM` mirror the measured values explicitly -/// for telemetry-aware clients. `submissionId` is the best-BPB champion row so -/// clients can open the detail modal; era / benches are filled by detail fan-out. +/// Ranks by **lattice score** descending (G2 equal-weight mean under +/// `scoring_version` 4) — never min-bpb alone. Per hotkey, the highest-scoring +/// champion row wins. `elo` carries the lattice score so the existing row +/// contract can surface rankings; `bpb` / `paramsM` remain measured secondary +/// fields. `submissionId` opens the detail modal; era / benches fill via +/// detail fan-out. #[must_use] pub fn prism_bpb_leaderboard(subs: &[Value], epoch: u64) -> Vec { - let mut best: HashMap, String)> = HashMap::new(); + // (score, bpb, n_params, submission_id) — higher score wins per hotkey. + let mut best: HashMap, Option, String)> = HashMap::new(); let mut counts: HashMap = HashMap::new(); for row in subs { if row.get("status").and_then(Value::as_str) != Some("terminated") { @@ -662,9 +664,17 @@ pub fn prism_bpb_leaderboard(subs: &[Value], epoch: u64) -> Vec if !is_prism_champion_submission(row) { continue; } - let Some(bpb) = row.get("bpb").and_then(Value::as_f64) else { + let Some(score) = row + .get("score") + .and_then(|s| s.get("value")) + .and_then(Value::as_u64) + else { continue; }; + if score == 0 { + continue; + } + let bpb = row.get("bpb").and_then(Value::as_f64); let n_params = row.get("n_params").and_then(Value::as_u64); let id = row .get("id") @@ -678,36 +688,38 @@ pub fn prism_bpb_leaderboard(subs: &[Value], epoch: u64) -> Vec .to_owned(); *counts.entry(hk.clone()).or_insert(0) += 1; best.entry(hk) - .and_modify(|(b, p, sid)| { - if bpb < *b { + .and_modify(|(s, b, p, sid)| { + if score > *s { + *s = score; *b = bpb; *p = n_params; sid.clone_from(&id); } }) - .or_insert((bpb, n_params, id)); + .or_insert((score, bpb, n_params, id)); } - let mut rows: Vec<(f64, String, u32, Option, String)> = best + let mut rows: Vec<_> = best .into_iter() - .map(|(hk, (bpb, n_params, sid))| { + .map(|(hk, (score, bpb, n_params, sid))| { let n = counts.get(&hk).copied().unwrap_or(1); - (bpb, hk, n, n_params, sid) + (score, bpb, hk, n, n_params, sid) }) .collect(); - rows.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + // Higher lattice score first; ties break by lexicographically smaller id. + rows.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.5.cmp(&b.5))); rows.into_iter() .enumerate() .map( - |(i, (bpb, hk, submissions, n_params, sid))| LeaderboardRow { + |(i, (score, bpb, hk, submissions, n_params, sid))| LeaderboardRow { rank: u32::try_from(i + 1).unwrap_or(u32::MAX), agent: agent_from_hotkey(&hk, epoch), - elo: bpb, + elo: score as f64, wins: 0, losses: 0, win_rate: 0.0, submissions, delta7d: 0.0, - bpb: Some(bpb), + bpb, params_m: n_params.map(|p| p as f64 / 1e6), weight: None, tao_per_day: None, @@ -1460,8 +1472,10 @@ mod tests { ]; let rows = prism_bpb_leaderboard(&subs, 3); assert_eq!(rows.len(), 2); - assert!((rows[0].bpb.unwrap() - 1.0).abs() < f64::EPSILON); + // Higher lattice score first (bb=200), not lower bpb. assert_eq!(rows[0].submission_id.as_deref(), Some("b")); + assert!((rows[0].elo - 200.0).abs() < f64::EPSILON); + assert!((rows[0].bpb.unwrap() - 1.0).abs() < f64::EPSILON); assert!(rows[0].params_m.is_none()); assert!((rows[1].params_m.unwrap() - 12.0).abs() < f64::EPSILON); assert_eq!(rows[1].submission_id.as_deref(), Some("a")); @@ -1563,19 +1577,22 @@ mod tests { } #[test] - fn prism_bpb_leaderboard_ranks_lower_first() { + fn prism_score_leaderboard_ranks_higher_first() { let subs = vec![ json!({"id":"a","status":"terminated","bpb":2.0,"miner_hotkey":"aa","score":{"kind":"score","value":10}}), json!({"id":"b","status":"terminated","bpb":1.0,"miner_hotkey":"bb","score":{"kind":"score","value":20}}), json!({"id":"c","status":"queued","bpb":0.1,"miner_hotkey":"cc","score":{"kind":"score","value":30}}), + // Same hotkey as aa — higher score should win the slot. json!({"id":"d","status":"terminated","bpb":0.5,"miner_hotkey":"aa","score":{"kind":"score","value":40}}), ]; let rows = prism_bpb_leaderboard(&subs, 3); assert_eq!(rows.len(), 2); + assert_eq!(rows[0].submission_id.as_deref(), Some("d")); + assert!((rows[0].elo - 40.0).abs() < f64::EPSILON); + assert_eq!(rows[1].submission_id.as_deref(), Some("b")); + assert!((rows[1].elo - 20.0).abs() < f64::EPSILON); assert_eq!(rows[0].rank, 1); - assert!((rows[0].elo - 0.5).abs() < f64::EPSILON); assert_eq!(rows[0].submissions, 2); - assert!((rows[1].elo - 1.0).abs() < f64::EPSILON); } #[test] diff --git a/docs/PRISM.md b/docs/PRISM.md index a39e48e6b..de941d400 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -222,8 +222,9 @@ lands in, not the leaf format or the math).** Per emitted epoch set: the epoch projects all-zero (burn / hold) — never emit Prism share to 1.x. **Top-model publish + secure receive.** The master tracks the global best -bpb across **weight-eligible** (recipe 2.0 / AutoModel) scored submissions. -After a successful Lium eval it +**lattice score** (G2 equal-weight accuracies under `scoring_version` 4 — +never min-bpb alone) across **weight-eligible** (recipe 2.0 / AutoModel) +scored submissions. After a successful Lium eval it **pulls** `checkpoint.pt` from the pod over SSH (master-initiated; the pod never pushes) and stages it through the secure receive hook into `$PRISM_ARTIFACT_DIR//` **before** terminate. Staging @@ -251,7 +252,8 @@ check is tighter when measured params are known. Oversized payloads are refused **before** writing. Top-model publish calls `verify_parked` and refuses weights without a -valid receipt. On a new global best (≤ best ever and < last published), it +valid receipt. On a new global-best lattice score (≥ best ever and > +last published score), it publishes `architecture.py` + `training.py` + `METRICS.json` + `ARTIFACT.json` + a `README.md` block to the public [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) repo diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index e1c4c1042..0708ab615 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -225,8 +225,8 @@ set emitted at the first chain-epoch boundary **after** your run finalizes (a long train that crosses epochs is normal — outbox assignment is exactly once). Positive scores then keep participating in later epochs' competition sets until a better valid score supersedes them (WTA still collapses to one leaf winner). -The global-best model (sources + `ARTIFACT.json` / checkpoint release) is -published to +The global-best model by **G2 lattice score** (sources + `ARTIFACT.json` / +checkpoint release) is published to [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) `top-model/` and (when configured) a HuggingFace model repo `BaseIntelligence/top-prism-architecture` (custom-arch / AutoModel novelty +