Free the playground's GPU models before training; show why a run failed

frontend/src/pages/Runs.tsx

@@ -228,6 +228,15 @@ function RunDetail({ run }: { run: JobSummary }) {
           <Stat label="Learning rate" value={last ? last.lr.toExponential(1) : undefined} />
         </StatStrip>
 
+        {job?.error && (
+          <section className="border border-destructive/30 bg-destructive/5 p-4">
+            <h3 className="mb-2 flex items-center gap-1.5 text-sm font-semibold text-destructive">
+              <CircleAlert className="size-4" /> Why it failed
+            </h3>
+            <pre className="max-h-40 overflow-auto font-mono text-xs whitespace-pre-wrap text-destructive">{job.error}</pre>
+          </section>
+        )}
+
         <Tabs value={tab} onValueChange={(v) => setTab(v as typeof tab)} className="gap-4">
           <TabsList variant="line" className="w-full justify-start border-b border-border">
             <TabsTrigger value="loss" className="flex-none">Loss curves</TabsTrigger>
@@ -272,14 +281,6 @@ function RunDetail({ run }: { run: JobSummary }) {
           </TabsContent>
 
           <TabsContent value="artifact" className="space-y-4">
-            {job?.error && (
-              <section className="border border-destructive/30 bg-destructive/5 p-4">
-                <h3 className="mb-2 flex items-center gap-1.5 text-sm font-semibold text-destructive">
-                  <CircleAlert className="size-4" /> Error
-                </h3>
-                <pre className="font-mono text-xs whitespace-pre-wrap text-destructive">{job.error}</pre>
-              </section>
-            )}
             {job?.checkpoint ? (
               <Panel title="Trained adapter">
                 <div className="flex items-center justify-between gap-4 px-4 py-3 text-sm">

shadowlm/_static/index.html

@@ -11,7 +11,7 @@
       href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700&family=Sora:wght@500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap"
     />
     <title>openfinetuner</title>
-    <script type="module" crossorigin src="./assets/index-C5gMH3FJ.js"></script>
+    <script type="module" crossorigin src="./assets/index-ChIABspn.js"></script>
     <link rel="stylesheet" crossorigin href="./assets/index-sV2ScQ20.css">
   </head>
   <body>

shadowlm/serve.py

@@ -774,6 +774,14 @@ def _worker(self) -> None:
                     payload["dataset"] = {"rows": train.rows, "format": train.format}
                     payload["eval_dataset"] = {"rows": ev.rows, "format": ev.format}
                 with self._model_lock:
+                    # Playground models cached on the GPU would leave a big base
+                    # too little room to train (an 8B in bf16 next to a cached
+                    # 8B is an OOM on 48 GB), so a run starts on a clean card.
+                    if self._infer_cache:
+                        n, _ = self._drop_inference_models()
+                        print(f"[{job_id[:8]}] unloaded {n} playground "
+                              f"model{'s' if n != 1 else ''} to free the GPU "
+                              "for training", flush=True)
                     be = select_backend(self.backend_name,
                                         accelerator=self.accelerator,
                                         device=self.device)
@@ -918,27 +926,33 @@ def _gpu_used_mb() -> int | None:
         except Exception:  # noqa: BLE001
             return None
 
-    def clear_vram(self) -> dict:
-        """Drop every cached inference model and release the GPU allocator's
-        cache — frees VRAM held after inference/compare without restarting the
-        server. Queued/running training is untouched (one job at a time)."""
+    def _drop_inference_models(self) -> tuple[int, str | None]:
+        """Empty the inference cache and release the GPU allocator's cache.
+        Caller holds ``_model_lock``. Returns (models dropped, release error)."""
         import gc  # noqa: PLC0415
 
-        before = self._gpu_used_mb()
-        freed_error: str | None = None
-        with self._model_lock:
-            n = len(self._infer_cache)
-            self._infer_cache.clear()
+        n = len(self._infer_cache)
+        self._infer_cache.clear()
         gc.collect()
+        freed_error: str | None = None
         try:
             import torch  # noqa: PLC0415
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
                 torch.cuda.synchronize()
-        except Exception as e:  # noqa: BLE001 — report, don't fail the request
+        except Exception as e:  # noqa: BLE001 — report, don't fail the caller
             freed_error = f"{type(e).__name__}: {e}"
             print(f"[serve] VRAM release failed ({freed_error})", flush=True)
         gc.collect()
+        return n, freed_error
+
+    def clear_vram(self) -> dict:
+        """Drop every cached inference model and release the GPU allocator's
+        cache — frees VRAM held after inference/compare without restarting the
+        server. Queued/running training is untouched (one job at a time)."""
+        before = self._gpu_used_mb()
+        with self._model_lock:
+            n, freed_error = self._drop_inference_models()
         out = {"unloaded": n, "before_mb": before, "after_mb": self._gpu_used_mb()}
         if freed_error:  # don't report a clean sweep when the allocator threw
             out["error"] = freed_error

tests/test_serve_vram.py

@@ -0,0 +1,29 @@
+"""Training starts on a clean card: cached playground models are dropped
+first, so a big base isn't fighting an inference copy of itself for VRAM.
+"""
+
+from __future__ import annotations
+
+from shadowlm.serve import Server
+
+
+def _server(tmp_path) -> Server:
+    return Server(backend="auto", accelerator="auto", device="auto", work_root=tmp_path)
+
+
+def test_drop_inference_models_empties_the_cache(tmp_path):
+    s = _server(tmp_path)
+    s._infer_cache[("Qwen/Qwen3-8B", None, None)] = object()
+    s._infer_cache[("Qwen/Qwen3-8B", "/a", None)] = object()
+    with s._model_lock:
+        n, err = s._drop_inference_models()
+    assert (n, err) == (2, None)
+    assert s._infer_cache == {}
+
+
+def test_clear_vram_reports_what_it_unloaded(tmp_path):
+    s = _server(tmp_path)
+    s._infer_cache[("m", None, None)] = object()
+    out = s.clear_vram()
+    assert out["unloaded"] == 1 and "error" not in out
+    assert s._infer_cache == {}