feat(prism): live leaf from G2 benchmarks (scoring_version 4)

bins/prism-challenge/Cargo.toml

@@ -32,6 +32,8 @@ prism-review = { path = "../../crates/prism-review" }
 submission-gating = { path = "../../crates/submission-gating" }
 crypto = { path = "../../crates/crypto" }
 db = { path = "../../crates/db" }
+serde_json = "1"
+sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "postgres"] }
 telemetry = { path = "../../crates/telemetry" }
 tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal", "sync", "time"] }
 trustroot = { path = "../../crates/trustroot" }

bins/prism-challenge/src/main.rs

@@ -26,8 +26,9 @@ use challenge_agentic::{AgenticBackend, OpenRouterAgent, SimAgent};
 use challenge_keys::load_challenge_secret;
 use clap::{Parser, Subcommand};
 use prism_challenge::{
-    submission_router, AppState, DbEvalStore, DbPrismStore, EvalStore, MemoryEvalStore,
-    MemoryPrismStore, Orchestrator, OrchestratorConfig, PrismStore, CHALLENGE_ID, SCORING_VERSION,
+    score_from_g2_benchmarks, submission_router, AppState, DbEvalStore, DbPrismStore, EvalStore,
+    FinalScore, MemoryEvalStore, MemoryPrismStore, Orchestrator, OrchestratorConfig, PrismStore,
+    Stage, StageEvent, StatePatch, CHALLENGE_ID,
 };
 use prism_lium::{EvalJobBackend, LiumClient, LiumSshConfig, SimLiumBackend};
 use prism_lium_payer::{allow_operator_lium_fallback, PayerBackendFactory, PayerKeyVault};
@@ -114,10 +115,22 @@ struct Cli {
 
 #[derive(Debug, Subcommand)]
 enum Cmd {
-    /// Print identity.
+    /// Print identity (id, live `scoring_version`/`mode`, public key).
     Identity,
     /// Run server + workers (default).
     Serve,
+    /// Recompute `final_score` from stored G2 metrics (v4). Requires `BASE_DATABASE_URL`.
+    RescoreG2 {
+        /// Print planned updates only.
+        #[arg(long, default_value_t = false)]
+        dry_run: bool,
+        /// Optional single submission id.
+        #[arg(long)]
+        id: Option<String>,
+        /// Max champion rows to scan.
+        #[arg(long, default_value_t = 500)]
+        limit: u32,
+    },
 }
 
 fn main() -> ExitCode {
@@ -139,8 +152,18 @@ fn main() -> ExitCode {
 }
 
 fn run(cli: Cli) -> Result<(), String> {
-    if matches!(cli.cmd, Some(Cmd::Identity)) {
-        return cmd_identity(&resolve_sk_path(cli.challenge_sk_file.as_ref())?);
+    match &cli.cmd {
+        Some(Cmd::Identity) => {
+            return cmd_identity(&resolve_sk_path(cli.challenge_sk_file.as_ref())?);
+        }
+        Some(Cmd::RescoreG2 { dry_run, id, limit }) => {
+            let rt = tokio::runtime::Builder::new_multi_thread()
+                .enable_all()
+                .build()
+                .map_err(|e| e.to_string())?;
+            return rt.block_on(cmd_rescore_g2(*dry_run, id.clone(), *limit));
+        }
+        _ => {}
     }
     let rt = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
@@ -166,11 +189,82 @@ fn cmd_identity(path: &Path) -> Result<(), String> {
     let sk = load_challenge_secret(path).map_err(|e| e.to_string())?;
     let pk = prism_challenge::public_key_from_secret(&sk)?;
     println!("challenge_id={CHALLENGE_ID}");
-    println!("scoring_version={SCORING_VERSION}");
+    let mode = prism_challenge::ScoringMode::from_env();
+    println!("scoring_version={}", mode.scoring_version());
+    println!("scoring_mode={}", mode.name());
     println!("public_key={}", encode_hex(&pk));
     Ok(())
 }
 
+async fn cmd_rescore_g2(dry_run: bool, id: Option<String>, limit: u32) -> Result<(), String> {
+    let url = std::env::var("BASE_DATABASE_URL")
+        .map_err(|_| "BASE_DATABASE_URL required for rescore-g2".to_string())?;
+    let pool = db::connect(&url).await.map_err(|e| e.to_string())?;
+    let store = DbPrismStore::new(pool.clone());
+    let rows = if let Some(id) = id.as_deref() {
+        match store.get(id).await.map_err(|e| e.to_string())? {
+            Some(r) => vec![r],
+            None => return Err(format!("unknown submission {id}")),
+        }
+    } else {
+        store
+            .list_champions(limit)
+            .await
+            .map_err(|e| e.to_string())?
+    };
+    let mut updated = 0u32;
+    let mut skipped = 0u32;
+    for row in rows {
+        let Some(metrics) = row.metrics_json.as_ref() else {
+            skipped += 1;
+            continue;
+        };
+        let new_score = score_from_g2_benchmarks(metrics);
+        let old = match row.final_score {
+            Some(FinalScore::Score(v)) => v,
+            _ => 0,
+        };
+        if old == new_score {
+            skipped += 1;
+            continue;
+        }
+        println!("{}: {old} -> {new_score}", row.id);
+        if dry_run {
+            updated += 1;
+            continue;
+        }
+        store
+            .apply(
+                &row.id,
+                &StatePatch {
+                    status: Some(Stage::Terminated),
+                    final_score: Some(FinalScore::Score(new_score)),
+                    ..StatePatch::default()
+                },
+                Some(&StageEvent {
+                    stage: Stage::Terminated,
+                    detail: Some(serde_json::json!({
+                        "rescore": "g2_benchmarks",
+                        "scoring_version": 4,
+                        "old_score": old,
+                        "new_score": new_score,
+                    })),
+                    at_ms: 0,
+                }),
+            )
+            .await
+            .map_err(|e| e.to_string())?;
+        sqlx::query("UPDATE prism_submission SET emitted_epoch = NULL WHERE id = $1")
+            .bind(&row.id)
+            .execute(&pool)
+            .await
+            .map_err(|e| e.to_string())?;
+        updated += 1;
+    }
+    println!("rescore-g2 done updated={updated} skipped={skipped} dry_run={dry_run}");
+    Ok(())
+}
+
 /// Resolve the Lium API key (file/env/credentials — never logged).
 fn load_lium_api_key() -> Option<String> {
     if let Ok(path) = std::env::var("LIUM_API_KEY_FILE") {
@@ -427,7 +521,7 @@ fn orchestrator_config(
         stuck_grace_secs: 10 * 3600,
         stage_delay,
         auto_retry_max: cli.auto_retry_max,
-        // scoring_mode from PRISM_SCORING_MODE (default shadow).
+        // scoring_mode from PRISM_SCORING_MODE (default benchmarks / v4 G2).
         ..Default::default()
     }
 }
@@ -536,8 +630,7 @@ async fn cmd_serve(cli: Cli) -> Result<(), String> {
     )
     .with_topmodel(topmodel)
     // v3: persist the METRICS_JSON v2 battery + compute the composite
-    // (scoring mode from PRISM_SCORING_MODE; default shadow keeps the v2
-    // score bit-identical).
+    // (scoring mode from PRISM_SCORING_MODE; default benchmarks = v4 G2).
     .with_eval_store(Some(eval_store))
     .with_runtime(Arc::clone(&active_jobs), Arc::clone(&log_buf));
     orchestrator = attach_miner_payer(orchestrator, payer_vault, live_ssh, operator_key.is_some());

crates/prism-challenge-task/src/lib.rs

@@ -1,10 +1,10 @@
 //! PRISM challenge identity constants.
 //!
-//! Normative pins (`scoring_version = 2`):
+//! Normative pins (`scoring_version = 4` live):
 //!
 //! ```text
 //! challenge_id     = "prism"
-//! scoring_version  = 2
+//! scoring_version  = 4   # G2 public-suite benchmarks (default)
 //! task_id domain   = b"base-prism-task-id-v1"
 //! task_blob domain = b"base-prism-task-blob-v1"
 //! answer domain    = b"base-prism-answer-v1"
@@ -22,25 +22,23 @@ pub const CHALLENGE_ID: &str = "prism";
 /// UTF-8 bytes of [`CHALLENGE_ID`].
 pub const CHALLENGE_ID_BYTES: &[u8] = b"prism";
 
-/// Live `challenge_scoring_version` for PRISM (integer score map).
+/// Legacy v2 `challenge_scoring_version` (pure bits/token bpb leaf).
 ///
-/// v1 blended measured bpb with an LLM quality vote (0.7/0.3). v2 drops the
-/// LLM vote from the number entirely: the score is **pure bpb**, and the
-/// LLM/coherence review is an anti-cheat GATE recording an audit event
-/// (`Copied`/`Suspicious` still hard-zero). Older v1 leaves stay attributable
-/// under their version tag.
+/// Retained for `PRISM_SCORING_MODE=shadow` rows and historical leaves.
 pub const SCORING_VERSION: u16 = 2;
 
 /// v3 composite scoring version (G1–G8 weighted geometric mean with
 /// lexicographic gates, clustered-bootstrap LCB lattice).
 ///
-/// **Not live**: v2 remains the live default until the anchor set is
-/// calibrated on the reference baselines and governance flips
-/// `PRISM_SCORING_MODE` from `shadow` to `composite`. Only rows scored under
-/// composite mode carry this version tag; the bundle `protocol_version`
-/// stays 1.
+/// Opt-in via `PRISM_SCORING_MODE=composite` after anchors are calibrated.
 pub const SCORING_VERSION_V3: u16 = 3;
 
+/// Live default: equal-weight G2 public-suite accuracies → lattice.
+///
+/// `PRISM_SCORING_MODE=benchmarks` (default). Tokenizer length no longer
+/// farms the emission leaf; see `prism_final::score_from_g2_benchmarks`.
+pub const SCORING_VERSION_V4: u16 = 4;
+
 /// Domain tag for PRISM task id digests.
 pub const TASK_ID_DOMAIN: &[u8] = b"base-prism-task-id-v1";
 
@@ -71,6 +69,8 @@ mod tests {
     #[test]
     fn scoring_version_and_score_max() {
         assert_eq!(SCORING_VERSION, 2);
+        assert_eq!(SCORING_VERSION_V3, 3);
+        assert_eq!(SCORING_VERSION_V4, 4);
         assert_eq!(SCORE_MAX, 1_000_000);
     }
 

crates/prism-challenge/src/api.rs

@@ -1488,7 +1488,7 @@ mod tests {
         .await;
         assert_eq!(s, StatusCode::OK);
         let eval = &v["submission"]["eval"];
-        assert_eq!(eval["scoring_mode"], "shadow");
+        assert_eq!(eval["scoring_mode"], "benchmarks");
         assert_eq!(
             eval["status"], "ineligible",
             "partial battery is ineligible"

crates/prism-challenge/src/lib.rs

@@ -1,9 +1,4 @@
-//! PRISM challenge orchestrator on Base.
-//!
-//! Master-centralized GPU eval via Lium (or Sim). **No Phala CVM.**
-//! Miner submit API mirrors agent/hypertraining shape; scores emit D24 leaves
-//! under `challenge_id = "prism"`.
-
+//! PRISM challenge orchestrator (Lium GPU eval; D24 leaves).
 #![forbid(unsafe_code)]
 #![allow(clippy::cast_precision_loss)]
 #![allow(clippy::cast_possible_truncation)]
@@ -28,17 +23,15 @@ pub use prism_challenge_task::{
 };
 pub use prism_emit::{EmitError, EmitSummary, EpochEmitter};
 pub use prism_eval_store::{DbEvalStore, MemoryEvalStore};
-pub use prism_store::eval::EvalStore;
-// 2×2 attribution matrix (module lives in prism-recipe beside SourceTree for
-// the per-crate LOC cap; conceptually prism_challenge::attribution).
-pub use prism_final::{combine_final, FinalOutcome};
+pub use prism_final::{combine_final, score_from_g2_benchmarks, FinalOutcome};
 pub use prism_pipeline::{
     example_automodel_request, example_valid_request, expand_zip_fields, run_eval_pipeline,
     run_sim_pipeline, score_from_bpb, score_from_pipeline, submission_id, validate, PipelineError,
-    PipelineInput, PipelineOutcome, PipelineResult, PrismConfig, QueuedSubmission,
+    PipelineInput, PipelineOutcome, PipelineResult, PrismConfig, QueuedSubmission, ScoringMode,
     SubmissionAccepted, SubmissionError, SubmissionId, SubmissionRequest, SubmissionService,
 };
 pub use prism_recipe::attribution;
+pub use prism_store::eval::EvalStore;
 pub use prism_store::{
     DbPrismStore, FinalScore, MemoryPrismStore, PrismStore, Stage, StageEvent, StatePatch,
     StoreError, SubmissionState,

crates/prism-challenge/src/orchestrator.rs

@@ -1,8 +1,4 @@
-//! Lium job orchestrator: claim→screen→review→pod→score; epoch emitter via
-//! [`Orchestrator::run_emitter`]. State in store; API is a projection.
-//!
-//! Pre-pod order is fail-closed: copy/static/similarity + LLM quality +
-//! agentic (sources) must pass before any Lium rent.
+//! Lium orchestrator: claim→screen→review→pod→score; emitter via `run_emitter`.
 
 use std::sync::Arc;
 use std::time::Duration;
@@ -421,8 +417,7 @@ impl<C: ChainClient + Send> Orchestrator<C> {
         }
         let bpb = metrics.as_ref().map(|m| m.bpb);
 
-        // Metrics-aware agentic pass (inconsistent_metrics / eval forge).
-        // Structural cheats already failed closed pre-pod — this never rents.
+        // Metrics-aware agentic (inconsistent_metrics / forge); never rents.
         let Some(agentic) = self
             .agentic_step(&id, &row, metrics.as_ref(), receipt.as_ref())
             .await
@@ -451,6 +446,7 @@ impl<C: ChainClient + Send> Orchestrator<C> {
                 similarity_evidence: similarity.evidence.clone(),
                 agentic: agentic.verdict,
                 composite,
+                metrics: blob.clone(),
             },
             None => FinalOutcome::ChallengeInternal,
         };

crates/prism-challenge/tests/agentic_review_retry.rs

@@ -16,7 +16,7 @@ use challenge_agentic::{AgenticBackend, AgenticError, AgenticVerdict, ReviewRequ
 use crypto::KEY_LEN;
 use prism_challenge::{
     FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator,
-    OrchestratorConfig, PrismStore, Stage, SubmissionState,
+    OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState,
 };
 use prism_lium::{
     EvalJobBackend, Instance, InstanceSpec, LiumError, Offer, RemoteExecResult, SimLiumBackend,
@@ -196,6 +196,7 @@ async fn agentic_infra_pre_pod_never_provisions() {
     let orch = Orchestrator::new(
         OrchestratorConfig {
             netuid: 541,
+            scoring_mode: ScoringMode::Shadow,
             auto_retry_max: 1,
             claim_poll: std::time::Duration::from_millis(10),
             ..Default::default()
@@ -281,6 +282,7 @@ async fn agentic_infra_retry_resumes_without_remeasure() {
     let orch = Orchestrator::new(
         OrchestratorConfig {
             netuid: 541,
+            scoring_mode: ScoringMode::Shadow,
             auto_retry_max: 1,
             claim_poll: std::time::Duration::from_millis(10),
             ..Default::default()

crates/prism-challenge/tests/arch_competition.rs

@@ -16,7 +16,7 @@ use challenge_agentic::SimAgent;
 use crypto::KEY_LEN;
 use prism_challenge::{
     FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator,
-    OrchestratorConfig, PrismStore, Stage, SubmissionState,
+    OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState,
 };
 use prism_lium::{EvalJobBackend, SimLiumBackend};
 use prism_review::SimReviewer;
@@ -147,6 +147,7 @@ fn mk_orchestrator(
     Orchestrator::new(
         OrchestratorConfig {
             netuid: 541,
+            scoring_mode: ScoringMode::Shadow,
             claim_poll: std::time::Duration::from_millis(10),
             ..Default::default()
         },

crates/prism-challenge/tests/cheat_arch_copy.rs

@@ -12,7 +12,7 @@ use challenge_agentic::SimAgent;
 use crypto::KEY_LEN;
 use prism_challenge::{
     FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator,
-    OrchestratorConfig, PrismStore, Stage, SubmissionState,
+    OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState,
 };
 use prism_lium::{EvalJobBackend, SimLiumBackend};
 use prism_recipe::{BASELINE_ARCHITECTURE_PY, BASELINE_TRAINING_PY};
@@ -93,6 +93,7 @@ async fn baseline_arch_train_copy_scores_zero() {
     let orch = Arc::new(Orchestrator::new(
         OrchestratorConfig {
             netuid: 541,
+            scoring_mode: ScoringMode::Shadow,
             claim_poll: std::time::Duration::from_millis(10),
             ..Default::default()
         },

crates/prism-challenge/tests/cheat_metrics.rs

@@ -12,7 +12,7 @@ use challenge_agentic::SimAgent;
 use crypto::KEY_LEN;
 use prism_challenge::{
     FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator,
-    OrchestratorConfig, PrismStore, Stage, SubmissionState,
+    OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState,
 };
 use prism_lium::{EvalJobBackend, SimLiumBackend};
 use prism_review::SimReviewer;
@@ -92,6 +92,7 @@ async fn hardcoded_metrics_json_scores_zero() {
     let orch = Arc::new(Orchestrator::new(
         OrchestratorConfig {
             netuid: 541,
+            scoring_mode: ScoringMode::Shadow,
             claim_poll: std::time::Duration::from_millis(10),
             ..Default::default()
         },

crates/prism-challenge/tests/copy_gate.rs

@@ -14,7 +14,7 @@ use challenge_agentic::SimAgent;
 use crypto::KEY_LEN;
 use prism_challenge::{
     FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator,
-    OrchestratorConfig, PrismStore, Stage, SubmissionState,
+    OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState,
 };
 use prism_lium::{EvalJobBackend, SimLiumBackend};
 use prism_review::SimReviewer;
@@ -139,6 +139,7 @@ fn mk_orchestrator(
     Orchestrator::new(
         OrchestratorConfig {
             netuid: 541,
+            scoring_mode: ScoringMode::Shadow,
             claim_poll: std::time::Duration::from_millis(10),
             ..Default::default()
         },