From e5753ec3dc0d6b9c2ed1c28893b4511beac347cb Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 12 Aug 2026 05:37:42 +0000 Subject: [PATCH] fix(design): log per-harness auto-enqueue skips Round-loop discarded EnqueueRoundResult.skipped, so quota/elimination failures had no operational signal. Warn each skip and document hex-hotkey miner lookups in external troubleshoot. --- crates/design-challenge/src/orchestrator.rs | 25 +++++++++++++++++++-- docs/external-miner/troubleshoot.md | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/design-challenge/src/orchestrator.rs b/crates/design-challenge/src/orchestrator.rs index 17097a7dc..a7c3c0e08 100644 --- a/crates/design-challenge/src/orchestrator.rs +++ b/crates/design-challenge/src/orchestrator.rs @@ -251,7 +251,7 @@ impl Orchestrator { // Open the round, then auto-enqueue every eligible active harness // with this round's shared prompt (Scheduled origin; idempotent). let _ = self.ensure_round(rid).await; - if let Err(e) = enqueue_active_harnesses_for_round( + match enqueue_active_harnesses_for_round( self.store.as_ref(), rid, self.cfg.netuid, @@ -259,7 +259,28 @@ impl Orchestrator { ) .await { - warn!(error = %e, round = rid, "auto-enqueue failed"); + Ok(result) => { + // Per-harness failures used to be discarded — ops had no signal + // when quota / elimination skipped an individual miner. + for (harness_id, miner_hotkey, reason) in &result.skipped { + warn!( + harness_id = %harness_id, + miner_hotkey = %miner_hotkey, + round = rid, + reason = %reason, + "auto-enqueue skipped harness" + ); + } + if !result.skipped.is_empty() { + info!( + round = rid, + scheduled = result.scheduled.len(), + skipped = result.skipped.len(), + "auto-enqueue finished with skips" + ); + } + } + Err(e) => warn!(error = %e, round = rid, "auto-enqueue failed"), } } } diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index f12ad5097..77408e0c7 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -10,7 +10,7 @@ |---------|--------------|---------------| | `400` on `POST /v1/harness` | Invalid bundle | `agent.py` defines `run`, `pyproject.toml` non-empty, size limits | | `409 schedule` "daily manual run quota exceeded" | Manual anti-spam cap (10/day) — round-loop auto-enqueue does **not** spend it | `GET /v1/quota/{hotkey}` → `manual.remaining`; wait until next UTC day | -| Active harness but no runs this round | Rare race / restart before auto-enqueue; or eliminated cooldown | Wait for the round tick / ask ops `admin/rounds/current/requeue`; check `eliminated_until_round` | +| Active harness but no runs this round | Checked before round-tick enqueue; not latest active harness on hotkey; scheduled quota; or queried with SS58 instead of lowercase hex | `GET /v1/miners/{hotkey}` and `GET /v1/quota/{hotkey}` with **64-char lowercase hex** (SS58 lowercases to a different key and shows empty); confirm harness `id` is the only `active: true` on that hotkey; `scheduled.remaining`; ops `admin/rounds/current/requeue` → `skipped[].reason` | | `auto_retry` events, class `install` | Dep won't install (bad name/version, heavy source build) | `GET /v1/runs/{id}/logs` phase `install`; fix `pyproject.toml` deps | | Run `failed` / Score 0 | Missing pages, timeout, crash | `GET /v1/runs/{id}/events`; ensure three required HTML pages | | External call refused (`403`) | Target is internal-blocklisted (metadata IP, loopback, RFC1918/VPC, control plane) | Call public endpoints only; egress is otherwise open |