fix(gateway): boot-seed backends so site-api survives restart

crates/db/migrations/0020_prism_eval_scoring_benchmarks.sql

@@ -0,0 +1,11 @@
+-- PRISM_SCORING_MODE default is `benchmarks` (v4 G2 lattice). Finalize writes
+-- that name into prism_eval_run.scoring_mode; the 0019 check only allowed
+-- shadow|composite, so every v4 finalize failed:
+--   new row for relation "prism_eval_run" violates check constraint
+--   "prism_eval_run_mode_check"
+-- Idempotent: prod may already have been patched by hand.
+
+ALTER TABLE prism_eval_run DROP CONSTRAINT IF EXISTS prism_eval_run_mode_check;
+ALTER TABLE prism_eval_run
+    ADD CONSTRAINT prism_eval_run_mode_check
+    CHECK (scoring_mode IN ('shadow', 'composite', 'benchmarks'));

crates/gateway-registry/src/lib.rs

@@ -358,6 +358,110 @@ impl Backend {
     }
 }
 
+/// Parse a boot-seed list: `challenge_id=url` entries separated by commas
+/// and/or newlines. Empty / whitespace-only input yields an empty vec.
+///
+/// Optional `;weight` suffix (default 1): `design=http://design:8093;2`.
+///
+/// # Errors
+///
+/// [`RegistryError::Invalid`] on malformed entries or bad URLs.
+pub fn parse_backend_seed_list(raw: &str) -> Result<Vec<CreateBackend>, RegistryError> {
+    let mut out = Vec::new();
+    for part in raw.split([',', '\n', '\r']) {
+        let part = part.trim();
+        if part.is_empty() {
+            continue;
+        }
+        let (challenge_id, rest) = part.split_once('=').ok_or_else(|| {
+            RegistryError::Invalid(format!(
+                "backend seed entry must be challenge_id=url[,…]; got `{part}`"
+            ))
+        })?;
+        let challenge_id = challenge_id.trim();
+        if challenge_id.is_empty() {
+            return Err(RegistryError::Invalid(
+                "backend seed challenge_id must be non-empty".into(),
+            ));
+        }
+        let (url_raw, weight) = match rest.rsplit_once(';') {
+            Some((url, w)) if w.trim().is_empty() => (url, 1u32),
+            Some((url, w)) if !w.contains("://") => {
+                let weight: u32 = w.trim().parse().map_err(|_| {
+                    RegistryError::Invalid(format!(
+                        "backend seed weight must be u32; got `{}`",
+                        w.trim()
+                    ))
+                })?;
+                (url, weight)
+            }
+            _ => (rest, 1u32),
+        };
+        out.push(CreateBackend {
+            challenge_id: challenge_id.to_owned(),
+            base_url: normalize_base_url(url_raw)?,
+            weight,
+        });
+    }
+    Ok(out)
+}
+
+/// `BASE_GATEWAY_BACKENDS` — comma/newline `challenge_id=url` list.
+pub const BACKENDS_ENV: &str = "BASE_GATEWAY_BACKENDS";
+/// Optional file whose contents are parsed like [`BACKENDS_ENV`] (wins when set).
+pub const BACKENDS_FILE_ENV: &str = "BASE_GATEWAY_BACKENDS_FILE";
+
+/// Load optional boot-seed backends from env / file.
+///
+/// # Errors
+///
+/// Unreadable file or malformed seed list.
+pub fn load_backend_seed_from_env() -> Result<Vec<CreateBackend>, RegistryError> {
+    if let Ok(path) = std::env::var(BACKENDS_FILE_ENV) {
+        let path = path.trim();
+        if !path.is_empty() {
+            let raw = std::fs::read_to_string(path).map_err(|e| {
+                RegistryError::Invalid(format!("read {BACKENDS_FILE_ENV} `{path}`: {e}"))
+            })?;
+            return parse_backend_seed_list(&raw);
+        }
+    }
+    match std::env::var(BACKENDS_ENV) {
+        Ok(raw) if !raw.trim().is_empty() => parse_backend_seed_list(&raw),
+        _ => Ok(Vec::new()),
+    }
+}
+
+impl Registry {
+    /// Insert seed backends; skip rows already present (`Duplicate`).
+    ///
+    /// Returns how many rows were newly created.
+    ///
+    /// # Errors
+    ///
+    /// Propagates non-duplicate [`RegistryError`] from [`Self::create`].
+    pub fn seed(&self, backends: &[CreateBackend]) -> Result<usize, RegistryError> {
+        let mut created = 0usize;
+        for req in backends {
+            match self.create(req) {
+                Ok(_) => created += 1,
+                Err(RegistryError::Duplicate { .. }) => {}
+                Err(e) => return Err(e),
+            }
+        }
+        Ok(created)
+    }
+
+    /// Seed from [`load_backend_seed_from_env`].
+    ///
+    /// # Errors
+    ///
+    /// Seed parse/load failures, or non-duplicate registry insert errors.
+    pub fn seed_from_env(&self) -> Result<usize, RegistryError> {
+        self.seed(&load_backend_seed_from_env()?)
+    }
+}
+
 fn normalize_base_url(raw: &str) -> Result<String, RegistryError> {
     let s = raw.trim().trim_end_matches('/').to_owned();
     if s.is_empty() {
@@ -544,4 +648,32 @@ mod tests {
         let picked = reg.pick("prism").expect("pick prism");
         assert_eq!(picked.base_url, "http://prism-challenge:8092");
     }
+
+    #[test]
+    fn parse_and_seed_compose_backends() {
+        let list = parse_backend_seed_list(
+            "prism=http://prism-challenge:8092, design=http://design-challenge:8093\n",
+        )
+        .expect("parse");
+        assert_eq!(list.len(), 2);
+        assert_eq!(list[0].challenge_id, "prism");
+        assert_eq!(list[0].base_url, "http://prism-challenge:8092");
+        assert_eq!(list[1].challenge_id, "design");
+        assert_eq!(list[1].weight, 1);
+
+        let reg = Registry::with_defaults();
+        assert_eq!(reg.seed(&list).expect("seed"), 2);
+        assert_eq!(reg.seed(&list).expect("idempotent"), 0);
+        assert_eq!(reg.list(None).len(), 2);
+        assert_eq!(
+            reg.pick("design").unwrap().base_url,
+            "http://design-challenge:8093"
+        );
+    }
+
+    #[test]
+    fn parse_backend_seed_rejects_bad_entry() {
+        let err = parse_backend_seed_list("not-a-pair").unwrap_err();
+        assert!(matches!(err, RegistryError::Invalid(_)));
+    }
 }

crates/gateway/src/lib.rs

@@ -399,6 +399,18 @@ where
     let metrics = init_metrics()?;
     // Prefer registry knobs from config when the shared handle was default-built.
     let _ = &config.registry;
+    // In-memory registry: seed from BASE_GATEWAY_BACKENDS(_FILE) so compose/prod
+    // restarts never leave /challenge/* at 503 until a manual admin POST.
+    let seeded = registry
+        .seed_from_env()
+        .map_err(|e| GatewayError::Config(format!("backend seed: {e}")))?;
+    if seeded > 0 {
+        tracing::info!(
+            event = "gateway_backends_seeded",
+            created = seeded,
+            "boot-seeded challenge backends into in-memory registry"
+        );
+    }
     let app = build_app(metrics, registry, chain, &config.tls, stores, extra)?;
 
     let listener = TcpListener::bind(config.listen)

deploy/AGENTS.md

@@ -40,7 +40,7 @@ Compose always runs a digest-pinned `postgres` service (`base-pgdata` volume, he
 | Gateway raw weight leaves + sealed bundles | **Postgres** (`raw_weight_snapshot`, `epoch_bundle`, …) |
 | Validator attestations (when DB configured) | **Postgres** |
 | Design sandbox staging files | volume `${BASE_STATE_DIR}/design/staging` + `design-artifacts` |
-| Gateway challenge **backend registry** | **in-memory** — re-seed after gateway restart (`remote-deploy.sh` does this on master) |
+| Gateway challenge **backend registry** | **in-memory**, boot-seeded from `BASE_GATEWAY_BACKENDS` (compose default: prism+design DNS URLs); `remote-deploy.sh` POST reseed stays idempotent |
 | site-api (`GET /v1/site/*`) | no DB — proxies challenge upstreams via gateway |
 | Unit/integration tests | may construct `Memory*Store` directly; omit `BASE_DATABASE_URL` only there |
 
@@ -104,6 +104,14 @@ install -m 0644 deploy/systemd/base-real-seal.{service,timer} /etc/systemd/syste
 systemctl daemon-reload && systemctl enable --now base-real-seal.timer
 ```
 
+**Challenge backend reseed (until boot-seed images are everywhere):** `base-reseed-backends.timer` (every **2 min**) drives [`scripts/prod-reseed-backends.sh`](scripts/prod-reseed-backends.sh) so a gateway restart cannot leave `/challenge/*` at 503 and `/v1/site/*` empty. Install on the master:
+
+```bash
+install -m 0755 deploy/scripts/prod-reseed-backends.sh /opt/base/deploy/scripts/prod-reseed-backends.sh
+install -m 0644 deploy/systemd/base-reseed-backends.{service,timer} /etc/systemd/system/
+systemctl daemon-reload && systemctl enable --now base-reseed-backends.timer
+```
+
 ## Chain endpoint failover (`BASE_CHAIN_ENDPOINTS`)
 
 Public Finney RPCs rate-limit per source IP (entrypoint-finney: HTTP 429 `http_60s` policy, or HTTP 200 with `"Too many requests from this source."`). Every Rust consumer (gateway, validator, both challenges, `weights-smoke`) goes through `chain-live`, which accepts an **ordered comma-separated endpoint list** and cools a faulted endpoint (429 / `-32005` / transport error) for 60s before retrying it; request-level JSON-RPC errors never fail over.

deploy/scripts/prod-reseed-backends.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Re-seed in-memory gateway challenge backends (idempotent).
+# The registry is wiped on every gateway restart; this heals 503s until
+# BASE_GATEWAY_BACKENDS boot-seed is in the running image.
+set -euo pipefail
+ROOT="${BASE_ROOT:-/opt/base}"
+cd "$ROOT"
+exec "$ROOT/deploy/scripts/register-challenge-backends.sh" \
+  --gateway-url http://127.0.0.1:8080 \
+  --prism-url http://prism-challenge:8092 \
+  --design-url http://design-challenge:8093

deploy/scripts/register-challenge-backends.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 # Register challenge reverse-proxy backends with the gateway registry.
 #
-# The in-memory registry is empty after every gateway restart/redeploy.
-# Call this on master after `docker compose up` (remote-deploy hooks it).
+# Prefer compose `BASE_GATEWAY_BACKENDS` (gateway boot-seeds on start).
+# This POST remains the fallback after a restart of an older image.
 #
 # Usage:
 #   GATEWAY_URL=http://127.0.0.1:8080 ./deploy/scripts/register-challenge-backends.sh

deploy/scripts/remote-deploy.sh

@@ -483,10 +483,9 @@ if [[ '$ROLE' == 'master' ]]; then
   else
     echo "gateway health: probe deferred"
   fi
-  # Registry is in-memory — re-seed challenge backends after every redeploy.
-  # The gateway races this script on boot, so retry until registration sticks,
-  # then prove proxy routing end-to-end: a missed reseed leaves /challenge/*
-  # at 503 while /healthz stays green. Both must fail the deploy loudly.
+  # Registry is in-memory; compose BASE_GATEWAY_BACKENDS boot-seeds on start.
+  # Still POST-reseed here: older images and a race on first listen leave
+  # /challenge/* at 503 while /healthz stays green. Both must fail loudly.
   # Gateway /v1/admin/* requires Authorization: Bearer (gateway_admin_token).
   echo "remote-deploy: registering challenge backends"
   reseed_ok=0

deploy/systemd/base-reseed-backends.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=BASE reseed gateway challenge backends (prism + design)
+After=docker.service network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/base/deploy/scripts/prod-reseed-backends.sh
+Nice=10
+TimeoutStartSec=60

deploy/systemd/base-reseed-backends.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Re-seed gateway challenge backends after restarts
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=2min
+Persistent=true
+
+[Install]
+WantedBy=timers.target

docker-compose.yml

@@ -93,6 +93,9 @@ services:
       BASE_TRUST_ROOT_DIR: ${BASE_TRUST_ROOT_DIR:-/etc/base/config}
       # Bundle seal mini-secret (host file, never baked into image)
       BASE_GATEWAY_SK_FILE: ${BASE_GATEWAY_SK_FILE:-/run/secrets/gateway_sk}
+      # Durable boot seed for the in-memory challenge registry (survives
+      # gateway restart). remote-deploy.sh POST reseed stays idempotent.
+      BASE_GATEWAY_BACKENDS: ${BASE_GATEWAY_BACKENDS:-prism=http://prism-challenge:8092,design=http://design-challenge:8093}
     volumes:
       - ./config:/etc/base/config:ro
       - ./deploy/secrets/gateway_sk:/run/secrets/gateway_sk:ro

docs/SITE_API.md

@@ -68,5 +68,6 @@ GPT-2 Large + Small constants live in `crates/site-api` (`prism_enrich`) so API
 `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle,
 slug, operator, and (for submissions) prompt title / id / run id.
 
-Backends must be registered (same as challenge proxy), e.g.
+Backends must be registered (same as challenge proxy). Compose
+boot-seeds them via `BASE_GATEWAY_BACKENDS`; manual fallback:
 `deploy/scripts/register-challenge-backends.sh`.