diff --git a/Cargo.lock b/Cargo.lock index a575ad565..9043f5c25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3869,6 +3869,8 @@ dependencies = [ "prism-recipe", "prism-registry", "prism-review", + "serde_json", + "sqlx", "submission-gating", "telemetry", "tempfile", diff --git a/bins/prism-challenge/Cargo.toml b/bins/prism-challenge/Cargo.toml index b5d062dbc..42a4e5015 100644 --- a/bins/prism-challenge/Cargo.toml +++ b/bins/prism-challenge/Cargo.toml @@ -32,6 +32,8 @@ prism-review = { path = "../../crates/prism-review" } submission-gating = { path = "../../crates/submission-gating" } crypto = { path = "../../crates/crypto" } db = { path = "../../crates/db" } +serde_json = "1" +sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "postgres"] } telemetry = { path = "../../crates/telemetry" } tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal", "sync", "time"] } trustroot = { path = "../../crates/trustroot" } diff --git a/bins/prism-challenge/src/main.rs b/bins/prism-challenge/src/main.rs index 51641d228..08cf6e24a 100644 --- a/bins/prism-challenge/src/main.rs +++ b/bins/prism-challenge/src/main.rs @@ -26,8 +26,9 @@ use challenge_agentic::{AgenticBackend, OpenRouterAgent, SimAgent}; use challenge_keys::load_challenge_secret; use clap::{Parser, Subcommand}; use prism_challenge::{ - submission_router, AppState, DbEvalStore, DbPrismStore, EvalStore, MemoryEvalStore, - MemoryPrismStore, Orchestrator, OrchestratorConfig, PrismStore, CHALLENGE_ID, SCORING_VERSION, + score_from_g2_benchmarks, submission_router, AppState, DbEvalStore, DbPrismStore, EvalStore, + FinalScore, MemoryEvalStore, MemoryPrismStore, Orchestrator, OrchestratorConfig, PrismStore, + Stage, StageEvent, StatePatch, CHALLENGE_ID, }; use prism_lium::{EvalJobBackend, LiumClient, LiumSshConfig, SimLiumBackend}; use prism_lium_payer::{allow_operator_lium_fallback, PayerBackendFactory, PayerKeyVault}; @@ -114,10 +115,22 @@ struct Cli { #[derive(Debug, Subcommand)] enum Cmd { - /// Print identity. + /// Print identity (id, live `scoring_version`/`mode`, public key). Identity, /// Run server + workers (default). Serve, + /// Recompute `final_score` from stored G2 metrics (v4). Requires `BASE_DATABASE_URL`. + RescoreG2 { + /// Print planned updates only. + #[arg(long, default_value_t = false)] + dry_run: bool, + /// Optional single submission id. + #[arg(long)] + id: Option, + /// Max champion rows to scan. + #[arg(long, default_value_t = 500)] + limit: u32, + }, } fn main() -> ExitCode { @@ -139,8 +152,18 @@ fn main() -> ExitCode { } fn run(cli: Cli) -> Result<(), String> { - if matches!(cli.cmd, Some(Cmd::Identity)) { - return cmd_identity(&resolve_sk_path(cli.challenge_sk_file.as_ref())?); + match &cli.cmd { + Some(Cmd::Identity) => { + return cmd_identity(&resolve_sk_path(cli.challenge_sk_file.as_ref())?); + } + Some(Cmd::RescoreG2 { dry_run, id, limit }) => { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + return rt.block_on(cmd_rescore_g2(*dry_run, id.clone(), *limit)); + } + _ => {} } let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -166,11 +189,82 @@ fn cmd_identity(path: &Path) -> Result<(), String> { let sk = load_challenge_secret(path).map_err(|e| e.to_string())?; let pk = prism_challenge::public_key_from_secret(&sk)?; println!("challenge_id={CHALLENGE_ID}"); - println!("scoring_version={SCORING_VERSION}"); + let mode = prism_challenge::ScoringMode::from_env(); + println!("scoring_version={}", mode.scoring_version()); + println!("scoring_mode={}", mode.name()); println!("public_key={}", encode_hex(&pk)); Ok(()) } +async fn cmd_rescore_g2(dry_run: bool, id: Option, limit: u32) -> Result<(), String> { + let url = std::env::var("BASE_DATABASE_URL") + .map_err(|_| "BASE_DATABASE_URL required for rescore-g2".to_string())?; + let pool = db::connect(&url).await.map_err(|e| e.to_string())?; + let store = DbPrismStore::new(pool.clone()); + let rows = if let Some(id) = id.as_deref() { + match store.get(id).await.map_err(|e| e.to_string())? { + Some(r) => vec![r], + None => return Err(format!("unknown submission {id}")), + } + } else { + store + .list_champions(limit) + .await + .map_err(|e| e.to_string())? + }; + let mut updated = 0u32; + let mut skipped = 0u32; + for row in rows { + let Some(metrics) = row.metrics_json.as_ref() else { + skipped += 1; + continue; + }; + let new_score = score_from_g2_benchmarks(metrics); + let old = match row.final_score { + Some(FinalScore::Score(v)) => v, + _ => 0, + }; + if old == new_score { + skipped += 1; + continue; + } + println!("{}: {old} -> {new_score}", row.id); + if dry_run { + updated += 1; + continue; + } + store + .apply( + &row.id, + &StatePatch { + status: Some(Stage::Terminated), + final_score: Some(FinalScore::Score(new_score)), + ..StatePatch::default() + }, + Some(&StageEvent { + stage: Stage::Terminated, + detail: Some(serde_json::json!({ + "rescore": "g2_benchmarks", + "scoring_version": 4, + "old_score": old, + "new_score": new_score, + })), + at_ms: 0, + }), + ) + .await + .map_err(|e| e.to_string())?; + sqlx::query("UPDATE prism_submission SET emitted_epoch = NULL WHERE id = $1") + .bind(&row.id) + .execute(&pool) + .await + .map_err(|e| e.to_string())?; + updated += 1; + } + println!("rescore-g2 done updated={updated} skipped={skipped} dry_run={dry_run}"); + Ok(()) +} + /// Resolve the Lium API key (file/env/credentials — never logged). fn load_lium_api_key() -> Option { if let Ok(path) = std::env::var("LIUM_API_KEY_FILE") { @@ -427,7 +521,7 @@ fn orchestrator_config( stuck_grace_secs: 10 * 3600, stage_delay, auto_retry_max: cli.auto_retry_max, - // scoring_mode from PRISM_SCORING_MODE (default shadow). + // scoring_mode from PRISM_SCORING_MODE (default benchmarks / v4 G2). ..Default::default() } } @@ -536,8 +630,7 @@ async fn cmd_serve(cli: Cli) -> Result<(), String> { ) .with_topmodel(topmodel) // v3: persist the METRICS_JSON v2 battery + compute the composite - // (scoring mode from PRISM_SCORING_MODE; default shadow keeps the v2 - // score bit-identical). + // (scoring mode from PRISM_SCORING_MODE; default benchmarks = v4 G2). .with_eval_store(Some(eval_store)) .with_runtime(Arc::clone(&active_jobs), Arc::clone(&log_buf)); orchestrator = attach_miner_payer(orchestrator, payer_vault, live_ssh, operator_key.is_some()); diff --git a/crates/prism-challenge-task/src/lib.rs b/crates/prism-challenge-task/src/lib.rs index f5337c84e..f71d4c098 100644 --- a/crates/prism-challenge-task/src/lib.rs +++ b/crates/prism-challenge-task/src/lib.rs @@ -1,10 +1,10 @@ //! PRISM challenge identity constants. //! -//! Normative pins (`scoring_version = 2`): +//! Normative pins (`scoring_version = 4` live): //! //! ```text //! challenge_id = "prism" -//! scoring_version = 2 +//! scoring_version = 4 # G2 public-suite benchmarks (default) //! task_id domain = b"base-prism-task-id-v1" //! task_blob domain = b"base-prism-task-blob-v1" //! answer domain = b"base-prism-answer-v1" @@ -22,25 +22,23 @@ pub const CHALLENGE_ID: &str = "prism"; /// UTF-8 bytes of [`CHALLENGE_ID`]. pub const CHALLENGE_ID_BYTES: &[u8] = b"prism"; -/// Live `challenge_scoring_version` for PRISM (integer score map). +/// Legacy v2 `challenge_scoring_version` (pure bits/token bpb leaf). /// -/// v1 blended measured bpb with an LLM quality vote (0.7/0.3). v2 drops the -/// LLM vote from the number entirely: the score is **pure bpb**, and the -/// LLM/coherence review is an anti-cheat GATE recording an audit event -/// (`Copied`/`Suspicious` still hard-zero). Older v1 leaves stay attributable -/// under their version tag. +/// Retained for `PRISM_SCORING_MODE=shadow` rows and historical leaves. pub const SCORING_VERSION: u16 = 2; /// v3 composite scoring version (G1–G8 weighted geometric mean with /// lexicographic gates, clustered-bootstrap LCB lattice). /// -/// **Not live**: v2 remains the live default until the anchor set is -/// calibrated on the reference baselines and governance flips -/// `PRISM_SCORING_MODE` from `shadow` to `composite`. Only rows scored under -/// composite mode carry this version tag; the bundle `protocol_version` -/// stays 1. +/// Opt-in via `PRISM_SCORING_MODE=composite` after anchors are calibrated. pub const SCORING_VERSION_V3: u16 = 3; +/// Live default: equal-weight G2 public-suite accuracies → lattice. +/// +/// `PRISM_SCORING_MODE=benchmarks` (default). Tokenizer length no longer +/// farms the emission leaf; see `prism_final::score_from_g2_benchmarks`. +pub const SCORING_VERSION_V4: u16 = 4; + /// Domain tag for PRISM task id digests. pub const TASK_ID_DOMAIN: &[u8] = b"base-prism-task-id-v1"; @@ -71,6 +69,8 @@ mod tests { #[test] fn scoring_version_and_score_max() { assert_eq!(SCORING_VERSION, 2); + assert_eq!(SCORING_VERSION_V3, 3); + assert_eq!(SCORING_VERSION_V4, 4); assert_eq!(SCORE_MAX, 1_000_000); } diff --git a/crates/prism-challenge/src/api.rs b/crates/prism-challenge/src/api.rs index 06daa494e..ddd90185e 100644 --- a/crates/prism-challenge/src/api.rs +++ b/crates/prism-challenge/src/api.rs @@ -1488,7 +1488,7 @@ mod tests { .await; assert_eq!(s, StatusCode::OK); let eval = &v["submission"]["eval"]; - assert_eq!(eval["scoring_mode"], "shadow"); + assert_eq!(eval["scoring_mode"], "benchmarks"); assert_eq!( eval["status"], "ineligible", "partial battery is ineligible" diff --git a/crates/prism-challenge/src/lib.rs b/crates/prism-challenge/src/lib.rs index 451b11289..b619bb8ba 100644 --- a/crates/prism-challenge/src/lib.rs +++ b/crates/prism-challenge/src/lib.rs @@ -1,9 +1,4 @@ -//! PRISM challenge orchestrator on Base. -//! -//! Master-centralized GPU eval via Lium (or Sim). **No Phala CVM.** -//! Miner submit API mirrors agent/hypertraining shape; scores emit D24 leaves -//! under `challenge_id = "prism"`. - +//! PRISM challenge orchestrator (Lium GPU eval; D24 leaves). #![forbid(unsafe_code)] #![allow(clippy::cast_precision_loss)] #![allow(clippy::cast_possible_truncation)] @@ -28,17 +23,15 @@ pub use prism_challenge_task::{ }; pub use prism_emit::{EmitError, EmitSummary, EpochEmitter}; pub use prism_eval_store::{DbEvalStore, MemoryEvalStore}; -pub use prism_store::eval::EvalStore; -// 2×2 attribution matrix (module lives in prism-recipe beside SourceTree for -// the per-crate LOC cap; conceptually prism_challenge::attribution). -pub use prism_final::{combine_final, FinalOutcome}; +pub use prism_final::{combine_final, score_from_g2_benchmarks, FinalOutcome}; pub use prism_pipeline::{ example_automodel_request, example_valid_request, expand_zip_fields, run_eval_pipeline, run_sim_pipeline, score_from_bpb, score_from_pipeline, submission_id, validate, PipelineError, - PipelineInput, PipelineOutcome, PipelineResult, PrismConfig, QueuedSubmission, + PipelineInput, PipelineOutcome, PipelineResult, PrismConfig, QueuedSubmission, ScoringMode, SubmissionAccepted, SubmissionError, SubmissionId, SubmissionRequest, SubmissionService, }; pub use prism_recipe::attribution; +pub use prism_store::eval::EvalStore; pub use prism_store::{ DbPrismStore, FinalScore, MemoryPrismStore, PrismStore, Stage, StageEvent, StatePatch, StoreError, SubmissionState, diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index 1f494d85e..3d0a640b2 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -1,8 +1,4 @@ -//! Lium job orchestrator: claim→screen→review→pod→score; epoch emitter via -//! [`Orchestrator::run_emitter`]. State in store; API is a projection. -//! -//! Pre-pod order is fail-closed: copy/static/similarity + LLM quality + -//! agentic (sources) must pass before any Lium rent. +//! Lium orchestrator: claim→screen→review→pod→score; emitter via `run_emitter`. use std::sync::Arc; use std::time::Duration; @@ -421,8 +417,7 @@ impl Orchestrator { } let bpb = metrics.as_ref().map(|m| m.bpb); - // Metrics-aware agentic pass (inconsistent_metrics / eval forge). - // Structural cheats already failed closed pre-pod — this never rents. + // Metrics-aware agentic (inconsistent_metrics / forge); never rents. let Some(agentic) = self .agentic_step(&id, &row, metrics.as_ref(), receipt.as_ref()) .await @@ -451,6 +446,7 @@ impl Orchestrator { similarity_evidence: similarity.evidence.clone(), agentic: agentic.verdict, composite, + metrics: blob.clone(), }, None => FinalOutcome::ChallengeInternal, }; diff --git a/crates/prism-challenge/tests/agentic_review_retry.rs b/crates/prism-challenge/tests/agentic_review_retry.rs index 1c4212d5f..b054ac3f9 100644 --- a/crates/prism-challenge/tests/agentic_review_retry.rs +++ b/crates/prism-challenge/tests/agentic_review_retry.rs @@ -16,7 +16,7 @@ use challenge_agentic::{AgenticBackend, AgenticError, AgenticVerdict, ReviewRequ use crypto::KEY_LEN; use prism_challenge::{ FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator, - OrchestratorConfig, PrismStore, Stage, SubmissionState, + OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState, }; use prism_lium::{ EvalJobBackend, Instance, InstanceSpec, LiumError, Offer, RemoteExecResult, SimLiumBackend, @@ -196,6 +196,7 @@ async fn agentic_infra_pre_pod_never_provisions() { let orch = Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, auto_retry_max: 1, claim_poll: std::time::Duration::from_millis(10), ..Default::default() @@ -281,6 +282,7 @@ async fn agentic_infra_retry_resumes_without_remeasure() { let orch = Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, auto_retry_max: 1, claim_poll: std::time::Duration::from_millis(10), ..Default::default() diff --git a/crates/prism-challenge/tests/arch_competition.rs b/crates/prism-challenge/tests/arch_competition.rs index dbafe4406..985e24e8e 100644 --- a/crates/prism-challenge/tests/arch_competition.rs +++ b/crates/prism-challenge/tests/arch_competition.rs @@ -16,7 +16,7 @@ use challenge_agentic::SimAgent; use crypto::KEY_LEN; use prism_challenge::{ FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator, - OrchestratorConfig, PrismStore, Stage, SubmissionState, + OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState, }; use prism_lium::{EvalJobBackend, SimLiumBackend}; use prism_review::SimReviewer; @@ -147,6 +147,7 @@ fn mk_orchestrator( Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, claim_poll: std::time::Duration::from_millis(10), ..Default::default() }, diff --git a/crates/prism-challenge/tests/cheat_arch_copy.rs b/crates/prism-challenge/tests/cheat_arch_copy.rs index c72ae847c..8010c4d14 100644 --- a/crates/prism-challenge/tests/cheat_arch_copy.rs +++ b/crates/prism-challenge/tests/cheat_arch_copy.rs @@ -12,7 +12,7 @@ use challenge_agentic::SimAgent; use crypto::KEY_LEN; use prism_challenge::{ FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator, - OrchestratorConfig, PrismStore, Stage, SubmissionState, + OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState, }; use prism_lium::{EvalJobBackend, SimLiumBackend}; use prism_recipe::{BASELINE_ARCHITECTURE_PY, BASELINE_TRAINING_PY}; @@ -93,6 +93,7 @@ async fn baseline_arch_train_copy_scores_zero() { let orch = Arc::new(Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, claim_poll: std::time::Duration::from_millis(10), ..Default::default() }, diff --git a/crates/prism-challenge/tests/cheat_metrics.rs b/crates/prism-challenge/tests/cheat_metrics.rs index a04f21fb7..449858e43 100644 --- a/crates/prism-challenge/tests/cheat_metrics.rs +++ b/crates/prism-challenge/tests/cheat_metrics.rs @@ -12,7 +12,7 @@ use challenge_agentic::SimAgent; use crypto::KEY_LEN; use prism_challenge::{ FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator, - OrchestratorConfig, PrismStore, Stage, SubmissionState, + OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState, }; use prism_lium::{EvalJobBackend, SimLiumBackend}; use prism_review::SimReviewer; @@ -92,6 +92,7 @@ async fn hardcoded_metrics_json_scores_zero() { let orch = Arc::new(Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, claim_poll: std::time::Duration::from_millis(10), ..Default::default() }, diff --git a/crates/prism-challenge/tests/copy_gate.rs b/crates/prism-challenge/tests/copy_gate.rs index b72681d75..0bb4b37f1 100644 --- a/crates/prism-challenge/tests/copy_gate.rs +++ b/crates/prism-challenge/tests/copy_gate.rs @@ -14,7 +14,7 @@ use challenge_agentic::SimAgent; use crypto::KEY_LEN; use prism_challenge::{ FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator, - OrchestratorConfig, PrismStore, Stage, SubmissionState, + OrchestratorConfig, PrismStore, ScoringMode, Stage, SubmissionState, }; use prism_lium::{EvalJobBackend, SimLiumBackend}; use prism_review::SimReviewer; @@ -139,6 +139,7 @@ fn mk_orchestrator( Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, claim_poll: std::time::Duration::from_millis(10), ..Default::default() }, diff --git a/crates/prism-challenge/tests/e2e_orchestrate_sim.rs b/crates/prism-challenge/tests/e2e_orchestrate_sim.rs index 4a00c1d55..d7d3c8bd5 100644 --- a/crates/prism-challenge/tests/e2e_orchestrate_sim.rs +++ b/crates/prism-challenge/tests/e2e_orchestrate_sim.rs @@ -13,7 +13,7 @@ use crypto::KEY_LEN; use prism_challenge::FinalScore; use prism_challenge::{ example_valid_request, submission_id, GatewayClient, GatewayClientConfig, MemoryPrismStore, - Orchestrator, OrchestratorConfig, PrismStore, Stage, StatePatch, SubmissionState, + Orchestrator, OrchestratorConfig, PrismStore, ScoringMode, Stage, StatePatch, SubmissionState, }; use prism_lium::{EvalJobBackend, SimLiumBackend}; use prism_review::SimReviewer; @@ -103,6 +103,7 @@ fn mk_orchestrator( Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, claim_poll: std::time::Duration::from_millis(10), ..Default::default() }, diff --git a/crates/prism-challenge/tests/e2e_v3_wiring.rs b/crates/prism-challenge/tests/e2e_v3_wiring.rs index a392591c7..411d2ed7f 100644 --- a/crates/prism-challenge/tests/e2e_v3_wiring.rs +++ b/crates/prism-challenge/tests/e2e_v3_wiring.rs @@ -14,7 +14,7 @@ use crypto::KEY_LEN; use prism_challenge::{ example_valid_request, submission_id, EvalStore, FinalScore, GatewayClient, GatewayClientConfig, MemoryEvalStore, MemoryPrismStore, Orchestrator, OrchestratorConfig, - PrismStore, Stage, SubmissionState, + PrismStore, ScoringMode, Stage, SubmissionState, }; use prism_lium::{ EvalJobBackend, Instance, InstanceSpec, LiumError, Offer, RemoteExecResult, SimLiumBackend, @@ -156,6 +156,7 @@ fn mk_orchestrator( Orchestrator::new( OrchestratorConfig { netuid: 541, + scoring_mode: ScoringMode::Shadow, claim_poll: std::time::Duration::from_millis(10), ..Default::default() }, diff --git a/crates/prism-eval-store/src/finalize.rs b/crates/prism-eval-store/src/finalize.rs index 0593a38f4..f1133f56d 100644 --- a/crates/prism-eval-store/src/finalize.rs +++ b/crates/prism-eval-store/src/finalize.rs @@ -364,10 +364,7 @@ async fn persist_run( CompositeOutcome::Scored(s) => &s.groups, CompositeOutcome::Ineligible(i) => &i.groups, }; - let scoring_mode = match ScoringMode::from_env() { - ScoringMode::Shadow => "shadow", - ScoringMode::Composite => "composite", - }; + let scoring_mode = ScoringMode::from_env().name(); let run = EvalRunRecord { run_id: String::new(), // store-assigned created_at: String::new(), // store-assigned @@ -553,7 +550,7 @@ mod tests { let run = st.eval_run("sub-2").await.unwrap().expect("run row"); assert_eq!(run.anchor_version, 0); - assert_eq!(run.scoring_mode, "shadow"); + assert_eq!(run.scoring_mode, "benchmarks"); assert_eq!(run.prereg_hash, AnchorInput::v0_placeholder().prereg_hash()); assert_eq!(run.eval_tier.as_deref(), Some("battery-v1")); assert_eq!(run.netns, Some(true)); @@ -573,7 +570,7 @@ mod tests { assert_eq!(prereg[0].hash, run.prereg_hash); let detail = eval_detail(&run, &groups); - assert_eq!(detail["scoring_mode"], "shadow"); + assert_eq!(detail["scoring_mode"], "benchmarks"); assert_eq!(detail["groups"].as_array().unwrap().len(), 8); assert!(detail["composite"].as_f64().unwrap() > 0.95); } diff --git a/crates/prism-eval-store/src/memory.rs b/crates/prism-eval-store/src/memory.rs index ce9923c98..f4d7bb2a6 100644 --- a/crates/prism-eval-store/src/memory.rs +++ b/crates/prism-eval-store/src/memory.rs @@ -232,7 +232,7 @@ mod tests { submission_id: sub.into(), anchor_version: 0, prereg_hash: "ff".repeat(32), - scoring_mode: "shadow".into(), + scoring_mode: "benchmarks".into(), pod_manifest: None, harness_files_sha256: None, netns: None, diff --git a/crates/prism-final/src/g2.rs b/crates/prism-final/src/g2.rs new file mode 100644 index 000000000..9bdd5da15 --- /dev/null +++ b/crates/prism-final/src/g2.rs @@ -0,0 +1,217 @@ +//! G2 public-suite lattice (scoring_version 4). +//! +//! Equal-weight mean of available downstream accuracies in `[0, 1]`, mapped to +//! `round(SCORE_MAX × mean)`. Tokenizer length cannot farm this leaf. +//! Missing every listed bench → `0` (fail-closed). Never falls back to bpb. + +use prism_challenge_task::SCORE_MAX; +use serde_json::Value; + +/// Canonical public G2 accuracy keys (first hit wins per task). +/// +/// Prefer Zone-A `org.g2.*`; accept harness battery aliases. LAMBADA prefers +/// the strict protocol when both MC and strict keys are present. +pub const G2_PUBLIC_BENCHES: &[(&str, &[&str])] = &[ + ( + "hellaswag", + &[ + "org.g2.hellaswag_acc", + "g2.hellaswag.acc_norm", + "g2.hellaswag.acc", + ], + ), + ( + "arc_easy", + &[ + "org.g2.arc_easy_acc", + "g2.arc_easy.acc_norm", + "g2.arc_easy.acc", + ], + ), + ( + "arc_challenge", + &[ + "org.g2.arc_challenge_acc", + "g2.arc_challenge.acc_norm", + "g2.arc_challenge.acc", + ], + ), + ( + "piqa", + &["org.g2.piqa_acc", "g2.piqa.acc_norm", "g2.piqa.acc"], + ), + ( + "winogrande", + &[ + "org.g2.winogrande_acc", + "g2.winogrande.acc_norm", + "g2.winogrande.acc", + ], + ), + ( + "boolq", + &["org.g2.boolq_acc", "g2.boolq.acc_norm", "g2.boolq.acc"], + ), + ( + "lambada", + &[ + "org.g2.lambada_strict_acc", + "g2.lambada_strict.acc", + "org.g2.lambada_acc", + "g2.lambada.acc_norm", + "g2.lambada.acc", + ], + ), + ( + "openbookqa", + &[ + "org.g2.obqa_acc", + "org.g2.openbookqa_acc", + "g2.openbookqa.acc_norm", + "g2.openbookqa.acc", + ], + ), +]; + +/// Extract finite accuracies in `[0, 1]` from a METRICS_JSON-shaped blob. +#[must_use] +pub fn extract_g2_accuracies(metrics: &Value) -> Vec<(&'static str, f64)> { + let mut out = Vec::new(); + for &(name, keys) in G2_PUBLIC_BENCHES { + if let Some(v) = first_acc(metrics, keys) { + out.push((name, v)); + } + } + out +} + +/// Equal-weight G2 mean → lattice. Empty suite → `0`. +#[must_use] +pub fn score_from_g2_benchmarks(metrics: &Value) -> u64 { + let accs = extract_g2_accuracies(metrics); + if accs.is_empty() { + return 0; + } + #[allow( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss + )] + { + let mean = accs.iter().map(|(_, a)| *a).sum::() / accs.len() as f64; + if !mean.is_finite() || mean <= 0.0 { + return 0; + } + let v = (mean * (SCORE_MAX as f64)).round(); + if v >= SCORE_MAX as f64 { + SCORE_MAX + } else { + v as u64 + } + } +} + +fn first_acc(metrics: &Value, keys: &[&str]) -> Option { + for key in keys { + if let Some(v) = lookup_f64(metrics, key) { + if v.is_finite() && (0.0..=1.0).contains(&v) { + return Some(v); + } + } + } + None +} + +fn lookup_f64(root: &Value, key: &str) -> Option { + if let Some(v) = as_finite_f64(root.get(key)) { + return Some(v); + } + // Nested org map (reference METRICS / some harvests): org. + if let Some(v) = as_finite_f64(root.pointer(&format!("/org/{key}"))) { + return Some(v); + } + // Flattened battery map: battery.metrics. + if let Some(v) = as_finite_f64(root.pointer(&format!("/battery/metrics/{key}"))) { + return Some(v); + } + // Zone-A row array: [{ "key", "value"|"point" }, ...] + if let Some(rows) = root.get("metrics").and_then(Value::as_array) { + for row in rows { + if row.get("key").and_then(Value::as_str) == Some(key) { + if let Some(v) = + as_finite_f64(row.get("value")).or_else(|| as_finite_f64(row.get("point"))) + { + return Some(v); + } + } + } + } + None +} + +fn as_finite_f64(v: Option<&Value>) -> Option { + let v = v?; + if let Some(n) = v.as_f64() { + return n.is_finite().then_some(n); + } + if let Some(obj) = v.as_object() { + return as_finite_f64(obj.get("value")).or_else(|| as_finite_f64(obj.get("point"))); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn equal_weight_mean_maps_to_lattice() { + let m = json!({ + "org.g2.hellaswag_acc": 0.5, + "org.g2.piqa_acc": 0.7, + }); + // mean 0.6 → 600_000 + assert_eq!(score_from_g2_benchmarks(&m), 600_000); + let names: Vec<_> = extract_g2_accuracies(&m) + .into_iter() + .map(|(n, _)| n) + .collect(); + assert_eq!(names, vec!["hellaswag", "piqa"]); + } + + #[test] + fn prefers_lambada_strict_over_mc() { + let m = json!({ + "org.g2.lambada_acc": 0.99, + "org.g2.lambada_strict_acc": 0.25, + }); + assert_eq!(score_from_g2_benchmarks(&m), 250_000); + } + + #[test] + fn empty_or_invalid_fail_closed() { + assert_eq!(score_from_g2_benchmarks(&json!({})), 0); + assert_eq!( + score_from_g2_benchmarks(&json!({"org.g2.hellaswag_acc": 1.5})), + 0 + ); + assert_eq!(score_from_g2_benchmarks(&json!({"bpb": 0.1})), 0); + } + + #[test] + fn reads_battery_nested_and_zone_a_rows() { + let nested = json!({"battery": {"metrics": {"org.g2.boolq_acc": {"value": 0.4}}}}); + assert_eq!(score_from_g2_benchmarks(&nested), 400_000); + let zone = json!({"metrics": [{"key": "org.g2.arc_easy_acc", "point": 0.8}]}); + assert_eq!(score_from_g2_benchmarks(&zone), 800_000); + } + + #[test] + fn prefers_obqa_alias_and_org_nest() { + let flat = json!({"org.g2.obqa_acc": 0.335}); + assert_eq!(score_from_g2_benchmarks(&flat), 335_000); + let nested = json!({"org": {"org.g2.obqa_acc": {"value": 0.5}}}); + assert_eq!(score_from_g2_benchmarks(&nested), 500_000); + } +} diff --git a/crates/prism-final/src/lib.rs b/crates/prism-final/src/lib.rs index 5d7483daa..3c5aa4729 100644 --- a/crates/prism-final/src/lib.rs +++ b/crates/prism-final/src/lib.rs @@ -7,61 +7,53 @@ #![allow(clippy::doc_markdown)] mod agentic; +mod g2; + pub use agentic::{build_review_request, corpus_from_rows, gate_corpus_from_rows, same_miner}; +pub use g2::{extract_g2_accuracies, score_from_g2_benchmarks, G2_PUBLIC_BENCHES}; use bundle::NoScoreReasonCode; use prism_pipeline::{final_lattice, CompositeOutcome, ScoringMode}; use prism_review::cheap_similarity_hard_zeros; +use serde_json::Value; -/// Final outcomes after LLM review + similarity + agentic gates (orchestrator v2). +/// Final outcomes after LLM review + similarity + agentic gates (orchestrator). #[derive(Debug, Clone, PartialEq)] pub enum FinalOutcome { - /// Measured bpb + LLM quality (0..1000) + similarity + agentic verdict. + /// Measured metrics + review gates. Leaf uses [`ScoringMode`]. Measured { - /// Bits-per-byte on the pinned val cut (lower is better). + /// Bits-per-byte on the pinned val cut (lower is better; audit / G1). bpb: f64, /// LLM quality verdict 0..1000. quality: u16, /// Cheap single-shot similarity class. similarity: prism_review::SimilarityKind, - /// Cheap LLM similarity confidence `0.0..1.0` (compared for - /// `Suspicious`; see [`prism_review::SUSPICIOUS_HARD_ZERO_THRESHOLD`]). + /// Cheap LLM similarity confidence `0.0..1.0`. similarity_score: f64, /// Cheap LLM evidence lines (trope-only → no Score wipe). similarity_evidence: Vec, /// Agentic anti-cheat verdict (primary gate). agentic: challenge_agentic::VerdictKind, /// v3 G1–G8 composite when the battery ran; `None` on the v2 path. - /// Under `PRISM_SCORING_MODE=shadow` (default) it never moves the - /// score; under `composite` its lattice becomes the score. composite: Option, + /// METRICS_JSON blob (G2 benches for [`ScoringMode::Benchmarks`]). + metrics: Option, }, /// Operator fault (any pipeline/review/similarity/agentic failure). ChallengeInternal, } -/// Map the measured outcome into the integer lattice under `mode` -/// (`OrchestratorConfig.scoring_mode`; v2 bit-identical under -/// [`ScoringMode::Shadow`]). -/// -/// The score is **pure bpb** in shadow mode: the LLM review is an -/// anti-cheat / coherence GATE, never a grader — its quality vote and issues -/// are recorded as audit events but never add nor remove points. +/// Map the measured outcome into the integer lattice under `mode`. /// -/// Hard gates (miner-attributable `Score{0}`), all checked *before* any -/// scoring-mode logic: -/// - Agentic `Cheat` / `Suspicious` (AST bands already applied upstream). -/// - Cheap LLM `Copied`. -/// - Cheap LLM `Suspicious` **only** when +/// Live default is [`ScoringMode::Benchmarks`] (G2 equal-weight accuracies). +/// Hard gates (miner-attributable `Score{0}`) run *before* scoring-mode logic: +/// - Agentic `Cheat` / `Suspicious` +/// - Cheap LLM `Copied` +/// - Cheap LLM `Suspicious` when /// `similarity_score >=` [`prism_review::SUSPICIOUS_HARD_ZERO_THRESHOLD`] -/// `(0.9)` **and** evidence is not generic-trope-only (RMSNorm / SwiGLU / -/// LayerNorm / …). Below the threshold (e.g. 0.7 tropes) → no score wipe. -/// -/// Missing agentic verdict is fail-closed upstream as -/// [`FinalOutcome::ChallengeInternal`]. +/// and evidence is not generic-trope-only /// -/// # Panics -/// Never. +/// Missing agentic is fail-closed upstream as [`FinalOutcome::ChallengeInternal`]. #[must_use] pub fn combine_final(outcome: &FinalOutcome, mode: ScoringMode) -> prism_store::FinalScore { use challenge_agentic::VerdictKind; @@ -77,6 +69,7 @@ pub fn combine_final(outcome: &FinalOutcome, mode: ScoringMode) -> prism_store:: similarity_evidence, agentic, composite, + metrics, .. } => { if matches!(agentic, VerdictKind::Cheat | VerdictKind::Suspicious) { @@ -85,7 +78,8 @@ pub fn combine_final(outcome: &FinalOutcome, mode: ScoringMode) -> prism_store:: if cheap_similarity_hard_zeros(*similarity, *similarity_score, similarity_evidence) { return FinalScore::Score(0); } - FinalScore::Score(final_lattice(*bpb, composite.as_ref(), mode)) + let g2 = metrics.as_ref().map(score_from_g2_benchmarks); + FinalScore::Score(final_lattice(*bpb, composite.as_ref(), mode, g2)) } } } @@ -97,6 +91,7 @@ mod final_tests { use prism_review::SimilarityKind::Copied; use prism_review::SimilarityKind::Original; use prism_review::SimilarityKind::Suspicious; + use serde_json::json; fn measured( similarity: prism_review::SimilarityKind, @@ -112,11 +107,10 @@ mod final_tests { similarity_evidence: evidence.iter().map(|s| (*s).to_owned()).collect(), agentic, composite: None, + metrics: None, } } - /// Clean-gate row at bpb 0.5 with a v3 composite attached (or not), for the - /// scoring-mode tests. `similarity` lets one case trip the v2 hard gate. fn measured_composite( similarity: prism_review::SimilarityKind, composite: Option, @@ -129,6 +123,7 @@ mod final_tests { similarity_evidence: vec![], agentic: challenge_agentic::VerdictKind::Clean, composite, + metrics: None, } } @@ -184,8 +179,6 @@ mod final_tests { #[test] fn hard_gates_precede_composite_scoring() { - // Even with a scored v3 composite attached, the v2 hard gates fire - // first, independent of scoring mode. let o = measured_composite(Copied, Some(scored_composite(999_999))); assert_eq!( combine_final(&o, ScoringMode::Composite), @@ -195,8 +188,6 @@ mod final_tests { #[test] fn shadow_mode_ignores_attached_composite() { - // Shadow (the default): the v2 number is bit-identical whether or - // not a composite is attached. let bare = measured_composite(Original, None); let with = measured_composite(Original, Some(scored_composite(1))); assert_eq!( @@ -216,7 +207,6 @@ mod final_tests { combine_final(&o, ScoringMode::Composite), prism_store::FinalScore::Score(777) ); - // Missing composite fails closed to 0 under composite mode. let bare = measured_composite(Original, None); assert_eq!( combine_final(&bare, ScoringMode::Composite), @@ -224,10 +214,44 @@ mod final_tests { ); } + #[test] + fn benchmarks_mode_uses_g2_never_bpb() { + let o = FinalOutcome::Measured { + bpb: 0.01, // would dominate under shadow + quality: 900, + similarity: Original, + similarity_score: 0.0, + similarity_evidence: vec![], + agentic: challenge_agentic::VerdictKind::Clean, + composite: None, + metrics: Some(json!({ + "org.g2.hellaswag_acc": 0.2, + "org.g2.piqa_acc": 0.4, + })), + }; + assert_eq!( + combine_final(&o, ScoringMode::Benchmarks), + prism_store::FinalScore::Score(300_000) + ); + let no_benches = FinalOutcome::Measured { + bpb: 0.01, + quality: 900, + similarity: Original, + similarity_score: 0.0, + similarity_evidence: vec![], + agentic: challenge_agentic::VerdictKind::Clean, + composite: None, + metrics: Some(json!({"bpb": 0.01})), + }; + assert_eq!( + combine_final(&no_benches, ScoringMode::Benchmarks), + prism_store::FinalScore::Score(0), + "no G2 → fail-closed; never bpb" + ); + } + #[test] fn quality_never_moves_the_score() { - // Anti-cheat review gates eligibility only; the quality vote must not - // shift the integer score by a single point. let hi = FinalOutcome::Measured { bpb: 0.5, quality: 900, @@ -236,6 +260,7 @@ mod final_tests { similarity_evidence: vec![], agentic: challenge_agentic::VerdictKind::Clean, composite: None, + metrics: None, }; let lo_same_bpb = FinalOutcome::Measured { bpb: 0.5, @@ -245,6 +270,7 @@ mod final_tests { similarity_evidence: vec![], agentic: challenge_agentic::VerdictKind::Clean, composite: None, + metrics: None, }; assert_eq!( combine_final(&hi, ScoringMode::Shadow), @@ -259,6 +285,7 @@ mod final_tests { similarity_evidence: vec![], agentic: challenge_agentic::VerdictKind::Clean, composite: None, + metrics: None, }; match ( combine_final(&hi, ScoringMode::Shadow), diff --git a/crates/prism-pipeline/src/lib.rs b/crates/prism-pipeline/src/lib.rs index f892df681..dc68cbfca 100644 --- a/crates/prism-pipeline/src/lib.rs +++ b/crates/prism-pipeline/src/lib.rs @@ -8,6 +8,7 @@ #![allow(clippy::cast_sign_loss)] #![allow(clippy::missing_errors_doc)] #![allow(clippy::doc_markdown)] +#![allow(clippy::must_use_candidate)] pub mod composite; pub mod config; diff --git a/crates/prism-pipeline/src/score.rs b/crates/prism-pipeline/src/score.rs index da6f0c1fe..e4a31c464 100644 --- a/crates/prism-pipeline/src/score.rs +++ b/crates/prism-pipeline/src/score.rs @@ -1,31 +1,19 @@ -//! Map BPB / pipeline outcomes to integer leaf scores (pipeline path). +//! Map BPB / pipeline outcomes / G2 benches to integer leaf scores. use std::sync::OnceLock; -use bundle::{NoScoreReasonCode, ScoreOrAbsence}; -use prism_challenge_task::{SCORE_MAX, SCORING_VERSION, SCORING_VERSION_V3}; - use crate::composite::CompositeOutcome; - +use bundle::{NoScoreReasonCode, ScoreOrAbsence}; +use prism_challenge_task::{SCORE_MAX, SCORING_VERSION, SCORING_VERSION_V3, SCORING_VERSION_V4}; /// Terminal measured outcome after master eval. #[derive(Debug, Clone, PartialEq)] pub enum PipelineOutcome { - /// Measured BPB (lower is better). Measured { bpb: f64 }, - /// Miner-attributable zero. MinerZero, - /// Operator / challenge fault. ChallengeInternal, - /// Explicit NoScore. NoScore { reason: NoScoreReasonCode }, - /// Already resolved. Resolved(ScoreOrAbsence), } - -/// Invert BPB into lattice score: lower bpb → higher score. -/// -/// Uses a soft map: `score = SCORE_MAX * (1 / (1 + bpb))` clamped. -#[must_use] pub fn score_from_bpb(bpb: f64) -> u64 { if !bpb.is_finite() || bpb < 0.0 { return 0; @@ -40,84 +28,55 @@ pub fn score_from_bpb(bpb: f64) -> u64 { v as u64 } } - -/// v3 scoring-mode selection, read once from `PRISM_SCORING_MODE` -/// (`shadow` default | `composite`). -/// -/// In `shadow` mode the composite (when present) is computed/stored by -/// callers but the leaf score stays `score_from_bpb` — v2 remains live until -/// anchors are calibrated and governance flips the mode. In `composite` mode -/// the v3 lattice is the leaf score and rows carry [`SCORING_VERSION_V3`] -/// (via [`ScoringMode::scoring_version`]). +/// `PRISM_SCORING_MODE`: default `benchmarks` (v4); `shadow`=v2 bpb; `composite`=v3. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ScoringMode { - /// v2 stays the score; v3 composite is observed only. Shadow, - /// v3 lattice is the score (fail-closed to 0 without a scored composite). Composite, + Benchmarks, } - impl ScoringMode { - /// Parse the raw env value: exactly `composite` selects composite mode. - #[must_use] pub fn parse(raw: Option<&str>) -> Self { - match raw { + match raw.map(str::trim) { + Some("shadow") => Self::Shadow, Some("composite") => Self::Composite, - _ => Self::Shadow, + _ => Self::Benchmarks, } } - - /// Mode for this process, read from `PRISM_SCORING_MODE` once. - #[must_use] pub fn from_env() -> Self { static MODE: OnceLock = OnceLock::new(); *MODE.get_or_init(|| Self::parse(std::env::var("PRISM_SCORING_MODE").ok().as_deref())) } - - /// `challenge_scoring_version` stamped on rows scored under this mode. - #[must_use] pub const fn scoring_version(self) -> u16 { match self { Self::Shadow => SCORING_VERSION, Self::Composite => SCORING_VERSION_V3, + Self::Benchmarks => SCORING_VERSION_V4, } } - - /// Stable label for logs, run rows, and terminal events. - #[must_use] pub const fn name(self) -> &'static str { match self { Self::Shadow => "shadow", Self::Composite => "composite", + Self::Benchmarks => "benchmarks", } } } - -/// Final integer lattice score after the (caller-applied) hard-zero gates, -/// honoring the scoring mode. -/// -/// - `shadow`: always the v2 `score_from_bpb` number (bit-identical). -/// - `composite`: the v3 lattice of an attached scored composite; an -/// ineligible or missing composite fails closed to 0 (emission burns). -#[must_use] -pub fn final_lattice(bpb: f64, composite: Option<&CompositeOutcome>, mode: ScoringMode) -> u64 { +pub fn final_lattice( + bpb: f64, + composite: Option<&CompositeOutcome>, + mode: ScoringMode, + g2_lattice: Option, +) -> u64 { match mode { ScoringMode::Shadow => score_from_bpb(bpb), ScoringMode::Composite => match composite { Some(CompositeOutcome::Scored(s)) => s.lattice, - Some(CompositeOutcome::Ineligible(_)) => 0, - None => { - tracing::warn!( - "composite scoring mode but no CompositeOutcome attached; scoring 0" - ); - 0 - } + _ => 0, }, + ScoringMode::Benchmarks => g2_lattice.unwrap_or(0), } } - -/// Map pipeline outcome to leaf payload. -#[must_use] pub fn score_from_pipeline(outcome: &PipelineOutcome) -> ScoreOrAbsence { match outcome { PipelineOutcome::Resolved(s) => s.clone(), @@ -131,7 +90,6 @@ pub fn score_from_pipeline(outcome: &PipelineOutcome) -> ScoreOrAbsence { }, } } - #[cfg(test)] mod tests { use super::*; @@ -151,23 +109,29 @@ mod tests { } #[test] - fn scoring_mode_parse_defaults_to_shadow() { - assert_eq!(ScoringMode::parse(None), ScoringMode::Shadow); + fn scoring_mode_parse_defaults_to_benchmarks() { + assert_eq!(ScoringMode::parse(None), ScoringMode::Benchmarks); + assert_eq!( + ScoringMode::parse(Some("benchmarks")), + ScoringMode::Benchmarks + ); assert_eq!(ScoringMode::parse(Some("shadow")), ScoringMode::Shadow); assert_eq!( ScoringMode::parse(Some("composite")), ScoringMode::Composite ); - assert_eq!(ScoringMode::parse(Some("COMPOSITE")), ScoringMode::Shadow); - assert_eq!(ScoringMode::parse(Some("garbage")), ScoringMode::Shadow); + assert_eq!( + ScoringMode::parse(Some("COMPOSITE")), + ScoringMode::Benchmarks + ); + assert_eq!(ScoringMode::parse(Some("garbage")), ScoringMode::Benchmarks); } #[test] fn scoring_version_marks_mode() { - assert_eq!(ScoringMode::Shadow.scoring_version(), SCORING_VERSION); - assert_eq!(ScoringMode::Composite.scoring_version(), SCORING_VERSION_V3); assert_eq!(ScoringMode::Shadow.scoring_version(), 2); assert_eq!(ScoringMode::Composite.scoring_version(), 3); + assert_eq!(ScoringMode::Benchmarks.scoring_version(), 4); } #[test] @@ -175,12 +139,11 @@ mod tests { let composite = sample_scored(424_242); for bpb in [0.0, 0.5, 1.0, 4.0] { assert_eq!( - final_lattice(bpb, Some(&composite), ScoringMode::Shadow), + final_lattice(bpb, Some(&composite), ScoringMode::Shadow, Some(1)), score_from_bpb(bpb), - "shadow ignores the composite" ); assert_eq!( - final_lattice(bpb, None, ScoringMode::Shadow), + final_lattice(bpb, None, ScoringMode::Shadow, None), score_from_bpb(bpb) ); } @@ -190,16 +153,24 @@ mod tests { fn composite_mode_uses_lattice_and_fails_closed() { let composite = sample_scored(424_242); assert_eq!( - final_lattice(4.0, Some(&composite), ScoringMode::Composite), + final_lattice(4.0, Some(&composite), ScoringMode::Composite, None), 424_242, - "composite mode emits the v3 lattice, not bpb" ); let ineligible = CompositeOutcome::Ineligible(sample_ineligible()); assert_eq!( - final_lattice(0.5, Some(&ineligible), ScoringMode::Composite), + final_lattice(0.5, Some(&ineligible), ScoringMode::Composite, None), 0 ); - assert_eq!(final_lattice(0.5, None, ScoringMode::Composite), 0); + assert_eq!(final_lattice(0.5, None, ScoringMode::Composite, None), 0); + } + + #[test] + fn benchmarks_mode_uses_g2_never_bpb() { + assert_eq!( + final_lattice(0.01, None, ScoringMode::Benchmarks, Some(123_456)), + 123_456 + ); + assert_eq!(final_lattice(0.01, None, ScoringMode::Benchmarks, None), 0); } fn sample_scored(lattice: u64) -> CompositeOutcome { diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 358c510ed..bb446525a 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -109,7 +109,8 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] | prism v3 harness | done (branch `prism-better`) | Multi-file harness package (`main.py` + `prismlib/`, miner code in `unshare --net` subprocess), seeded train stream with authoritative token counter, G6 probes, `prismlib/cheatguard.py` AST audit, METRICS_JSON v2, miner-chosen tokenizer, G5 RULER/BABILong/natural (pretrain-only), `RECIPE_VERSION 1.4.0`. | | prism v3 eval battery | done (branch `prism-better`) | G1–G8 under `harness/eval/` (intrinsic, downstream, recall, reasoning, long-context, curve, inference, stability) + `eval/public_dev/` anchors family + `tests/smoke_battery.py`. | | prism v3 two-phase pod flow | done (branch `prism-better`) | `PHASE_TRAIN_DONE` marker → post-train staging of private assets + secret seed (SSH on Lium, dir on Sim) → `.ready` gate (fail-closed) → eval phase. Private tier recorded as `eval_tier`. | -| prism v3 composite scoring | done, shadow-default (branch `prism-better`) | `prism-pipeline::composite` (anchor normalization, gates, weighted geometric mean, bootstrap CIs, LCB) + `ScoringMode` (`PRISM_SCORING_MODE`; `shadow` keeps v2 bit-identical, `composite` flips to `scoring_version 3`). Orchestrator persists the battery via `EvalStore` and attaches the composite; parameter-cap breach is terminal `Score(0)` (`CAP_EXCEEDED`). | +| prism v4 G2 benchmark leaf | **live default** | `PRISM_SCORING_MODE=benchmarks` (default) → equal-weight G2 public accuracies → `scoring_version` 4; never falls back to bits/token bpb. `prism-challenge rescore-g2` rewrites historical rows from stored `metrics_json`. | +| prism v3 composite scoring | done, opt-in | `prism-pipeline::composite` + `ScoringMode::Composite` (`PRISM_SCORING_MODE=composite` → `scoring_version` 3). Orchestrator persists the battery via `EvalStore`; parameter-cap breach is terminal `Score(0)` (`CAP_EXCEEDED`). | | prism v3 eval store + Zone B | done (branch `prism-better`) | Migration 0017 (7 tables), `prism_store::eval::EvalStore`, memory + Postgres impls (`prism-eval-store`), composite finalization glue, `prism-zoneb` contract types; Zone B validated, labelled, never scored. | | prism v3 attribution | done (branch `prism-better`) | `prism_recipe::attribution` 2×2 matrix builder + `POST /v1/submissions/{id}/attribution` returning run plans as JSON (operator-triggered execution); the route lives in `crates/prism-attribution` (split for the per-crate LOC cap). | | prism v3 baselines | done (branch `prism-better`) | `crates/prism-recipe/baselines/` Transformer++ + hybrid delta reference trees embedded as `prism_recipe::baselines`; G1 scored keys promoted to tokenizer-neutral `org.g1.bits_per_byte_*`; `harness/eval/calibrate_anchors.py` fills references from baseline METRICS; numeric refs in `anchors/v0.json` remain **placeholder** until E6 GPU measurement + pre-registration. | diff --git a/docs/PRISM.md b/docs/PRISM.md index 0eb05a156..a39e48e6b 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -1,7 +1,7 @@ # PRISM challenge (Base) **challenge_id:** `prism` -**scoring_version:** `2` live (bpb-only; v1 blended a 0.3 LLM quality vote; the architecture competition below reallocates credits *inside* this same lattice — no chain-facing version change). **v3 addition (opt-in):** composite scoring ships behind `PRISM_SCORING_MODE` (`shadow` default — v2 score bit-identical, composite observed; `composite` — v3 lattice becomes the score, rows carry `scoring_version 3`). See **v3 composite scoring** below. +**scoring_version:** `4` live (equal-weight **G2 public-suite accuracies** → lattice; tokenizer length no longer farms the leaf). Legacy: `2` = pure bits/token bpb (`PRISM_SCORING_MODE=shadow`); `3` = full G1–G8 composite (`composite`, anchors required). Default mode is `benchmarks`. See **v4 G2 benchmark scoring** and **v3 composite scoring** below. **recipe_version:** `2.0.0` (pinned NeMo AutoModel base + miner unified diff; legacy **1.x** two-script / source-tree layouts rejected on live — see [`PRISM_RECIPE.md`](PRISM_RECIPE.md)) **port:** `8092` **emission_share_bps:** `5000` (equal split with `design`; sum `10000`) @@ -32,8 +32,9 @@ violation (`missing_telemetry_hooks` → `Score(0)`, terminal). Cheap `score ≥ 0.9` (`SUSPICIOUS_HARD_ZERO_THRESHOLD`) and evidence is not generic-trope-only. Agentic is the primary anti-cheat judge and must not treat standard LM components as plagiarism. The LLM quality vote is a -**coherence gate, never a grader**: the final score is pure bpb, with -hard-zero on agentic `cheat`/`suspicious` and cheap `Copied` / high-confidence +**coherence gate, never a grader**: the live leaf is the **v4 G2 benchmark +lattice** (equal-weight mean of public-suite accuracies), with hard-zero on +agentic `cheat`/`suspicious` and cheap `Copied` / high-confidence `Suspicious`. Missing agentic verdict is fail-closed (`ChallengeInternal`). Leaves are D24-complete per chain epoch, emitted at epoch close from the finalized-since-last-epoch batch (see **Leaf emission** @@ -274,17 +275,49 @@ Operators may re-stage via `POST /v1/admin/artifacts/{id}/receive` (same admin Bearer; requires `X-Prism-Sha256`; `n_params` from store or `X-Prism-N-Params`) — never an open pod upload. -## v3 composite scoring (versioned addition — opt-in) +## v4 G2 benchmark scoring (live default) -Everything in this section is a **versioned addition**: the live leaf score -stays v2 pure-bpb until governance flips `PRISM_SCORING_MODE=composite` -after the placeholder anchors are measured on the E6 baselines and -hash-committed. Modes (`prism-pipeline::ScoringMode`): +**Breaking change vs v2:** the emission leaf is no longer +`score_from_bpb` (bits/token). Tokenizer length cannot farm the rank. +`PRISM_SCORING_MODE` modes (`prism-pipeline::ScoringMode`): | Mode | Leaf score | `scoring_version` on rows | |------|-----------|---------------------------| -| `shadow` (default; the v2/"legacy" behavior) | v2 `score_from_bpb` — **bit-identical** | `2` | -| `composite` | v3 lattice (fail-closed `0` without a scored composite) | `3` | +| `benchmarks` (**default**) | equal-weight mean of available G2 public accuracies → `round(SCORE_MAX × mean)` | `4` | +| `shadow` | v2 `score_from_bpb` (legacy; bits/token) | `2` | +| `composite` | v3 G1–G8 lattice (fail-closed `0` without a scored composite) | `3` | + +**Formula.** From METRICS_JSON / Zone-A `org.g2.*` (battery aliases accepted), +take each present accuracy in `[0, 1]` among: + +`hellaswag`, `arc_easy`, `arc_challenge`, `piqa`, `winogrande`, `boolq`, +`lambada` (prefer `org.g2.lambada_strict_acc`), `openbookqa`. + +Equal-weight mean over the **available subset** (missing tasks are omitted, +not zero-filled). Empty suite → **`Score(0)`** (fail-closed). **Never** falls +back to bits/token bpb for the leaf. Bits/token bpb and tokenizer-neutral +`org.g1.bits_per_byte_*` remain recorded for display / future composite; they +do not move the v4 lattice. + +**Historical rows.** Terminal rows already scored under v2 keep their stored +`final_score` until an operator re-score. Recompute from stored +`metrics_json` without re-renting GPUs: + +```bash +prism-challenge rescore-g2 --dry-run # plan +prism-challenge rescore-g2 # apply (clears emitted_epoch) +prism-challenge rescore-g2 --id +``` + +Requires `BASE_DATABASE_URL`. Already-sealed epoch leaves stay; the next +epoch-close outbox picks up the new lattice. + +## v3 composite scoring (versioned addition — opt-in) + +Everything in this section remains a **versioned addition** behind +`PRISM_SCORING_MODE=composite` after placeholder anchors are measured on the +E6 baselines and hash-committed. The live default is **v4 benchmarks** (above), +not shadow bpb. **Source-tree submissions (v3 / recipe 1.3–1.4 historical).** Under recipe ≤ 1.4.0, miners could submit a full source tree as a ZIP (`zip_base64` or @@ -498,7 +531,7 @@ for the bpb score (coherence gate, never a grader). | Crate | Role | |-------|------| -| `prism-challenge-task` | Identity constants / domains (`SCORING_VERSION` 2, `SCORING_VERSION_V3` 3) | +| `prism-challenge-task` | Identity constants / domains (`SCORING_VERSION` 2, `SCORING_VERSION_V3` 3, `SCORING_VERSION_V4` 4 live) | | `prism-lium-types` | Lium data contract: error taxonomy, provider shapes, pod telemetry series, signed `EvalReceipt` + `NoScoreGate` | | `prism-lium` | Lium REST client, real recipe exec over SSH, post-train asset staging, `SimLiumBackend`; re-exports `prism-lium-types` | | `prism-recipe` | Contract validation, dataset pin, multi-file harness + G1–G8 battery + cheatguard, baseline sources, source-tree intake (`zip_submit`), attribution, anchor sets, v3 baselines | diff --git a/docs/external-miner/README.md b/docs/external-miner/README.md index 7add783b7..1c26db896 100644 --- a/docs/external-miner/README.md +++ b/docs/external-miner/README.md @@ -14,7 +14,7 @@ over HTTP to the live challenges: | Challenge | `challenge_id` | Scoring | Guide | Public miner repo | |-----------|----------------|---------|-------|-------------------| | Design | `design` | `challenge_scoring_version` **2** (daily share ≥2 wins + agentic) | [design.md](./design.md) | [BaseIntelligence/design-challenge](https://github.com/BaseIntelligence/design-challenge) | -| Prism | `prism` | `challenge_scoring_version` **2** (bpb-only) | [prism.md](./prism.md) | [BaseIntelligence/prism](https://github.com/BaseIntelligence/prism) | +| Prism | `prism` | `challenge_scoring_version` **4** (G2 public-suite benchmarks) | [prism.md](./prism.md) | [BaseIntelligence/prism](https://github.com/BaseIntelligence/prism) | Do **not** conflate version axes: diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index 7b78c962a..e1c4c1042 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -3,7 +3,7 @@ # Prism challenge — HTTP AutoModel patch submit **challenge_id:** `prism` -**scoring_version:** `2` live (bpb leaf; LLM review is an anti-cheat gate, not a grader). **v3 harness (default):** every scored run executes the **G1–G8 battery** (leaf score stays bpb while `PRISM_SCORING_MODE=shadow`); see *v3 scoring* below. +**scoring_version:** `4` live (equal-weight G2 public-suite accuracies → lattice; LLM review is an anti-cheat gate, not a grader). **v3 harness (default):** every scored run executes the **G1–G8 battery**; the leaf uses G2 benches while `PRISM_SCORING_MODE=benchmarks` (default). Legacy `shadow` = bits/token bpb; `composite` = full G1–G8 lattice when anchors are ready. **recipe_version:** `2.0.0` (pinned [NeMo AutoModel](https://github.com/NVIDIA-NeMo/Automodel) base + miner unified diff; legacy 1.x layouts rejected on live) **Path:** HTTP only — **no Phala/CVM** @@ -201,8 +201,11 @@ submission and never rents a Lium pod. ## Scoring (summary) -Final leaf score is pure bits-per-byte (bpb) on the lattice `[0, SCORE_MAX]`. -The shared **agentic** gate (AST + metrics/receipt) hard-zeros `cheat` / +Final leaf score (live `scoring_version` **4**) is the **equal-weight mean of +available G2 public accuracies** mapped to `round(SCORE_MAX × mean)` — not +bits/token bpb. Tokenizer length cannot farm the rank. Bits/token bpb and +tokenizer-neutral `bits_per_byte` remain recorded for display / G1. The shared +**agentic** gate (AST + metrics/receipt) hard-zeros `cheat` / `suspicious`. Cheap LLM similarity hard-zeros `Copied`, and `Suspicious` only when confidence `≥ 0.9` with non-generic evidence (below that — e.g. 0.7 citing RMSNorm/SwiGLU/LayerNorm — does **not** wipe your score). Copy/similarity @@ -214,7 +217,7 @@ quality is coherence-only, not a grader. Public gallery/leaderboard show champions only. **Competition (temporary):** emission uses **your own best training score only** — architecture-owner credit (rewarding arch owners when others train -well on their code) is **disabled** for now so the best-BPB trainer keeps +well on their code) is **disabled** for now so the best-scoring trainer keeps Prism's weights. Emission remains **winner-take-all**: only the single highest own score that epoch receives Prism's share (50% of the subnet); ties break by lexicographically smallest hotkey. Scores first land in the leaf @@ -229,7 +232,7 @@ published to `BaseIntelligence/top-prism-architecture` (custom-arch / AutoModel novelty + weights, `trust_remote_code`). See [`PRISM.md`](../PRISM.md). -## v3 scoring (shadow-by-default) +## v3 scoring (battery always; leaf mode via env) Recipe ≥ 1.3.0 harnesses run a **two-phase pod flow**: your code trains (`phase=train`), checkpoints, and only then does the operator stage private @@ -269,12 +272,14 @@ terminal-loss band) and the cross-miner cohort, and land a stored verdict (`ok` / `flagged` / `quarantined`) — verdicts are evidence, never an auto-zero. Malformed or over-cap envelopes reject `422` and store nothing. -While `PRISM_SCORING_MODE=shadow` (default) the leaf score stays pure bpb -(**bits/token** = CE / ln 2 — tokenizer-dependent; byte-level vocabs can look -artificially strong). Tokenizer-neutral `bits_per_byte` is recorded on every -run and already feeds G1; a shadow-leaf switch to bits/byte is deferred until -an explicit scoring-version / governance change (see [`PRISM.md`](../PRISM.md) -§ shadow leaf unit). After the reference baselines (**Transformer++** and +While `PRISM_SCORING_MODE=benchmarks` (default) the leaf score is the +**equal-weight mean of available G2 public accuracies** (HellaSwag, ARC-E/C, +PIQA, WinoGrande, BoolQ, LAMBADA strict when present, OpenBookQA), mapped to +`round(SCORE_MAX × mean)`. Missing every listed bench → `0` (fail-closed). +**Tokenizer length no longer farms the rank** — bits/token bpb is still +recorded (and tokenizer-neutral `bits_per_byte` feeds G1) but does **not** +drive emission. `PRISM_SCORING_MODE=shadow` restores legacy pure bits/token +bpb (v2). After the reference baselines (**Transformer++** and **hybrid delta** — published in-repo under `crates/prism-recipe/baselines/`) are measured and the anchor set is pre-registered, governance may flip to `composite`: group scores are anchor-normalized (**arithmetic** mean within