diff --git a/crates/db/migrations/0020_prism_eval_scoring_benchmarks.sql b/crates/db/migrations/0020_prism_eval_scoring_benchmarks.sql new file mode 100644 index 000000000..535902746 --- /dev/null +++ b/crates/db/migrations/0020_prism_eval_scoring_benchmarks.sql @@ -0,0 +1,11 @@ +-- PRISM_SCORING_MODE default is `benchmarks` (v4 G2 lattice). Finalize writes +-- that name into prism_eval_run.scoring_mode; the 0019 check only allowed +-- shadow|composite, so every v4 finalize failed: +-- new row for relation "prism_eval_run" violates check constraint +-- "prism_eval_run_mode_check" +-- Idempotent: prod may already have been patched by hand. + +ALTER TABLE prism_eval_run DROP CONSTRAINT IF EXISTS prism_eval_run_mode_check; +ALTER TABLE prism_eval_run + ADD CONSTRAINT prism_eval_run_mode_check + CHECK (scoring_mode IN ('shadow', 'composite', 'benchmarks')); diff --git a/crates/gateway-registry/src/lib.rs b/crates/gateway-registry/src/lib.rs index a5184ac6d..78c6a3113 100644 --- a/crates/gateway-registry/src/lib.rs +++ b/crates/gateway-registry/src/lib.rs @@ -358,6 +358,110 @@ impl Backend { } } +/// Parse a boot-seed list: `challenge_id=url` entries separated by commas +/// and/or newlines. Empty / whitespace-only input yields an empty vec. +/// +/// Optional `;weight` suffix (default 1): `design=http://design:8093;2`. +/// +/// # Errors +/// +/// [`RegistryError::Invalid`] on malformed entries or bad URLs. +pub fn parse_backend_seed_list(raw: &str) -> Result, RegistryError> { + let mut out = Vec::new(); + for part in raw.split([',', '\n', '\r']) { + let part = part.trim(); + if part.is_empty() { + continue; + } + let (challenge_id, rest) = part.split_once('=').ok_or_else(|| { + RegistryError::Invalid(format!( + "backend seed entry must be challenge_id=url[,…]; got `{part}`" + )) + })?; + let challenge_id = challenge_id.trim(); + if challenge_id.is_empty() { + return Err(RegistryError::Invalid( + "backend seed challenge_id must be non-empty".into(), + )); + } + let (url_raw, weight) = match rest.rsplit_once(';') { + Some((url, w)) if w.trim().is_empty() => (url, 1u32), + Some((url, w)) if !w.contains("://") => { + let weight: u32 = w.trim().parse().map_err(|_| { + RegistryError::Invalid(format!( + "backend seed weight must be u32; got `{}`", + w.trim() + )) + })?; + (url, weight) + } + _ => (rest, 1u32), + }; + out.push(CreateBackend { + challenge_id: challenge_id.to_owned(), + base_url: normalize_base_url(url_raw)?, + weight, + }); + } + Ok(out) +} + +/// `BASE_GATEWAY_BACKENDS` — comma/newline `challenge_id=url` list. +pub const BACKENDS_ENV: &str = "BASE_GATEWAY_BACKENDS"; +/// Optional file whose contents are parsed like [`BACKENDS_ENV`] (wins when set). +pub const BACKENDS_FILE_ENV: &str = "BASE_GATEWAY_BACKENDS_FILE"; + +/// Load optional boot-seed backends from env / file. +/// +/// # Errors +/// +/// Unreadable file or malformed seed list. +pub fn load_backend_seed_from_env() -> Result, RegistryError> { + if let Ok(path) = std::env::var(BACKENDS_FILE_ENV) { + let path = path.trim(); + if !path.is_empty() { + let raw = std::fs::read_to_string(path).map_err(|e| { + RegistryError::Invalid(format!("read {BACKENDS_FILE_ENV} `{path}`: {e}")) + })?; + return parse_backend_seed_list(&raw); + } + } + match std::env::var(BACKENDS_ENV) { + Ok(raw) if !raw.trim().is_empty() => parse_backend_seed_list(&raw), + _ => Ok(Vec::new()), + } +} + +impl Registry { + /// Insert seed backends; skip rows already present (`Duplicate`). + /// + /// Returns how many rows were newly created. + /// + /// # Errors + /// + /// Propagates non-duplicate [`RegistryError`] from [`Self::create`]. + pub fn seed(&self, backends: &[CreateBackend]) -> Result { + let mut created = 0usize; + for req in backends { + match self.create(req) { + Ok(_) => created += 1, + Err(RegistryError::Duplicate { .. }) => {} + Err(e) => return Err(e), + } + } + Ok(created) + } + + /// Seed from [`load_backend_seed_from_env`]. + /// + /// # Errors + /// + /// Seed parse/load failures, or non-duplicate registry insert errors. + pub fn seed_from_env(&self) -> Result { + self.seed(&load_backend_seed_from_env()?) + } +} + fn normalize_base_url(raw: &str) -> Result { let s = raw.trim().trim_end_matches('/').to_owned(); if s.is_empty() { @@ -544,4 +648,32 @@ mod tests { let picked = reg.pick("prism").expect("pick prism"); assert_eq!(picked.base_url, "http://prism-challenge:8092"); } + + #[test] + fn parse_and_seed_compose_backends() { + let list = parse_backend_seed_list( + "prism=http://prism-challenge:8092, design=http://design-challenge:8093\n", + ) + .expect("parse"); + assert_eq!(list.len(), 2); + assert_eq!(list[0].challenge_id, "prism"); + assert_eq!(list[0].base_url, "http://prism-challenge:8092"); + assert_eq!(list[1].challenge_id, "design"); + assert_eq!(list[1].weight, 1); + + let reg = Registry::with_defaults(); + assert_eq!(reg.seed(&list).expect("seed"), 2); + assert_eq!(reg.seed(&list).expect("idempotent"), 0); + assert_eq!(reg.list(None).len(), 2); + assert_eq!( + reg.pick("design").unwrap().base_url, + "http://design-challenge:8093" + ); + } + + #[test] + fn parse_backend_seed_rejects_bad_entry() { + let err = parse_backend_seed_list("not-a-pair").unwrap_err(); + assert!(matches!(err, RegistryError::Invalid(_))); + } } diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 53cc0c8e6..164cf0b67 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -399,6 +399,18 @@ where let metrics = init_metrics()?; // Prefer registry knobs from config when the shared handle was default-built. let _ = &config.registry; + // In-memory registry: seed from BASE_GATEWAY_BACKENDS(_FILE) so compose/prod + // restarts never leave /challenge/* at 503 until a manual admin POST. + let seeded = registry + .seed_from_env() + .map_err(|e| GatewayError::Config(format!("backend seed: {e}")))?; + if seeded > 0 { + tracing::info!( + event = "gateway_backends_seeded", + created = seeded, + "boot-seeded challenge backends into in-memory registry" + ); + } let app = build_app(metrics, registry, chain, &config.tls, stores, extra)?; let listener = TcpListener::bind(config.listen) diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index ab285878b..9cf55f9d0 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -40,7 +40,7 @@ Compose always runs a digest-pinned `postgres` service (`base-pgdata` volume, he | Gateway raw weight leaves + sealed bundles | **Postgres** (`raw_weight_snapshot`, `epoch_bundle`, …) | | Validator attestations (when DB configured) | **Postgres** | | Design sandbox staging files | volume `${BASE_STATE_DIR}/design/staging` + `design-artifacts` | -| Gateway challenge **backend registry** | **in-memory** — re-seed after gateway restart (`remote-deploy.sh` does this on master) | +| Gateway challenge **backend registry** | **in-memory**, boot-seeded from `BASE_GATEWAY_BACKENDS` (compose default: prism+design DNS URLs); `remote-deploy.sh` POST reseed stays idempotent | | site-api (`GET /v1/site/*`) | no DB — proxies challenge upstreams via gateway | | Unit/integration tests | may construct `Memory*Store` directly; omit `BASE_DATABASE_URL` only there | @@ -104,6 +104,14 @@ install -m 0644 deploy/systemd/base-real-seal.{service,timer} /etc/systemd/syste systemctl daemon-reload && systemctl enable --now base-real-seal.timer ``` +**Challenge backend reseed (until boot-seed images are everywhere):** `base-reseed-backends.timer` (every **2 min**) drives [`scripts/prod-reseed-backends.sh`](scripts/prod-reseed-backends.sh) so a gateway restart cannot leave `/challenge/*` at 503 and `/v1/site/*` empty. Install on the master: + +```bash +install -m 0755 deploy/scripts/prod-reseed-backends.sh /opt/base/deploy/scripts/prod-reseed-backends.sh +install -m 0644 deploy/systemd/base-reseed-backends.{service,timer} /etc/systemd/system/ +systemctl daemon-reload && systemctl enable --now base-reseed-backends.timer +``` + ## Chain endpoint failover (`BASE_CHAIN_ENDPOINTS`) Public Finney RPCs rate-limit per source IP (entrypoint-finney: HTTP 429 `http_60s` policy, or HTTP 200 with `"Too many requests from this source."`). Every Rust consumer (gateway, validator, both challenges, `weights-smoke`) goes through `chain-live`, which accepts an **ordered comma-separated endpoint list** and cools a faulted endpoint (429 / `-32005` / transport error) for 60s before retrying it; request-level JSON-RPC errors never fail over. diff --git a/deploy/scripts/prod-reseed-backends.sh b/deploy/scripts/prod-reseed-backends.sh new file mode 100755 index 000000000..2b983002c --- /dev/null +++ b/deploy/scripts/prod-reseed-backends.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Re-seed in-memory gateway challenge backends (idempotent). +# The registry is wiped on every gateway restart; this heals 503s until +# BASE_GATEWAY_BACKENDS boot-seed is in the running image. +set -euo pipefail +ROOT="${BASE_ROOT:-/opt/base}" +cd "$ROOT" +exec "$ROOT/deploy/scripts/register-challenge-backends.sh" \ + --gateway-url http://127.0.0.1:8080 \ + --prism-url http://prism-challenge:8092 \ + --design-url http://design-challenge:8093 diff --git a/deploy/scripts/register-challenge-backends.sh b/deploy/scripts/register-challenge-backends.sh index d3d2e14a3..b9464678e 100755 --- a/deploy/scripts/register-challenge-backends.sh +++ b/deploy/scripts/register-challenge-backends.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Register challenge reverse-proxy backends with the gateway registry. # -# The in-memory registry is empty after every gateway restart/redeploy. -# Call this on master after `docker compose up` (remote-deploy hooks it). +# Prefer compose `BASE_GATEWAY_BACKENDS` (gateway boot-seeds on start). +# This POST remains the fallback after a restart of an older image. # # Usage: # GATEWAY_URL=http://127.0.0.1:8080 ./deploy/scripts/register-challenge-backends.sh diff --git a/deploy/scripts/remote-deploy.sh b/deploy/scripts/remote-deploy.sh index 775aa4fb8..ee9fe7671 100755 --- a/deploy/scripts/remote-deploy.sh +++ b/deploy/scripts/remote-deploy.sh @@ -483,10 +483,9 @@ if [[ '$ROLE' == 'master' ]]; then else echo "gateway health: probe deferred" fi - # Registry is in-memory — re-seed challenge backends after every redeploy. - # The gateway races this script on boot, so retry until registration sticks, - # then prove proxy routing end-to-end: a missed reseed leaves /challenge/* - # at 503 while /healthz stays green. Both must fail the deploy loudly. + # Registry is in-memory; compose BASE_GATEWAY_BACKENDS boot-seeds on start. + # Still POST-reseed here: older images and a race on first listen leave + # /challenge/* at 503 while /healthz stays green. Both must fail loudly. # Gateway /v1/admin/* requires Authorization: Bearer (gateway_admin_token). echo "remote-deploy: registering challenge backends" reseed_ok=0 diff --git a/deploy/systemd/base-reseed-backends.service b/deploy/systemd/base-reseed-backends.service new file mode 100644 index 000000000..d631acd11 --- /dev/null +++ b/deploy/systemd/base-reseed-backends.service @@ -0,0 +1,10 @@ +[Unit] +Description=BASE reseed gateway challenge backends (prism + design) +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/opt/base/deploy/scripts/prod-reseed-backends.sh +Nice=10 +TimeoutStartSec=60 diff --git a/deploy/systemd/base-reseed-backends.timer b/deploy/systemd/base-reseed-backends.timer new file mode 100644 index 000000000..5ee278fb0 --- /dev/null +++ b/deploy/systemd/base-reseed-backends.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Re-seed gateway challenge backends after restarts + +[Timer] +OnBootSec=30s +OnUnitActiveSec=2min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/docker-compose.yml b/docker-compose.yml index 25c8570ac..0e6f1c163 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,6 +93,9 @@ services: BASE_TRUST_ROOT_DIR: ${BASE_TRUST_ROOT_DIR:-/etc/base/config} # Bundle seal mini-secret (host file, never baked into image) BASE_GATEWAY_SK_FILE: ${BASE_GATEWAY_SK_FILE:-/run/secrets/gateway_sk} + # Durable boot seed for the in-memory challenge registry (survives + # gateway restart). remote-deploy.sh POST reseed stays idempotent. + BASE_GATEWAY_BACKENDS: ${BASE_GATEWAY_BACKENDS:-prism=http://prism-challenge:8092,design=http://design-challenge:8093} volumes: - ./config:/etc/base/config:ro - ./deploy/secrets/gateway_sk:/run/secrets/gateway_sk:ro diff --git a/docs/SITE_API.md b/docs/SITE_API.md index 25dbaa911..94eff4a77 100644 --- a/docs/SITE_API.md +++ b/docs/SITE_API.md @@ -68,5 +68,6 @@ GPT-2 Large + Small constants live in `crates/site-api` (`prism_enrich`) so API `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle, slug, operator, and (for submissions) prompt title / id / run id. -Backends must be registered (same as challenge proxy), e.g. +Backends must be registered (same as challenge proxy). Compose +boot-seeds them via `BASE_GATEWAY_BACKENDS`; manual fallback: `deploy/scripts/register-challenge-backends.sh`.