feat(harbor): the agent's first baseline eval is budget-free

vero/src/vero/harbor/protocol.py

@@ -51,6 +51,10 @@ class StatusSummary:
     submit_enabled: bool
     # per (split, dataset_id): tier + whether the agent may evaluate it + remaining budget
     splits: list[dict] = field(default_factory=list)
+    # the seeded baseline sha and whether its one budget-free reference eval
+    # is still available (None/False when the task has no recorded baseline)
+    base_commit: str | None = None
+    free_baseline_available: bool = False
 
     def to_dict(self) -> dict:
         return asdict(self)
@@ -98,6 +102,8 @@ def build_status(
     submit_enabled: bool,
     budget: dict[tuple[str, str], SplitBudget],
     split_accesses: list[SplitAccess],
+    base_commit: str | None = None,
+    free_baseline_available: bool = False,
 ) -> StatusSummary:
     """Build the agent-facing status from the budget ledger + split tiers.
 
@@ -117,4 +123,9 @@ def build_status(
                 "remaining_run_budget": b.remaining_run_budget,
             }
         )
-    return StatusSummary(submit_enabled=submit_enabled, splits=splits)
+    return StatusSummary(
+        submit_enabled=submit_enabled,
+        splits=splits,
+        base_commit=base_commit,
+        free_baseline_available=free_baseline_available,
+    )

vero/src/vero/harbor/serve.py

@@ -221,6 +221,7 @@ async def build_components(config: ServeConfig) -> tuple[EvaluationSidecar, Veri
         agent_volume=Path(config.agent_volume),
         admin_volume=Path(config.admin_volume),
         submit_enabled=config.submit_enabled,
+        base_commit=config.base_commit,
     )
     verifier = Verifier(
         engine=engine,

vero/src/vero/harbor/server.py

@@ -55,21 +55,48 @@ def __init__(
         agent_volume: Path,
         admin_volume: Path,
         submit_enabled: bool = False,
+        base_commit: str | None = None,
     ):
         self.engine = engine
         self.split_accesses = split_accesses
         self.agent_repo_path = Path(agent_repo_path)
         self.agent_volume = Path(agent_volume)
         self.admin_volume = Path(admin_volume)
         self.submit_enabled = submit_enabled
+        self.base_commit = base_commit
+        self._free_baseline_used = False
 
     # ------------------------------------------------------------------
     # Handlers (the HTTP layer resolves `admin` from auth and calls these)
     # ------------------------------------------------------------------
 
     async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> EvalSummary:
         sha = await self._transfer_commit(req.commit)
-        exp = await self.engine.evaluate(replace(req, commit=sha), admin=admin)
+        # The agent's FIRST eval of the seeded baseline is budget-free. The
+        # baseline is the reference every candidate is implicitly compared to,
+        # yet it can never win selection (auto_best excludes base_commit), so
+        # metering it forces a choice between optimizing blind and paying a
+        # budgeted eval for a commit that cannot be selected (observed live:
+        # an optimizer that skipped the reference could not tell a no-op edit
+        # from an improvement and quit with budget unspent). Capped at one:
+        # later baseline evals debit normally, so free compute is bounded.
+        free_baseline = (
+            not admin
+            and self.base_commit is not None
+            and sha == self.base_commit
+            and not self._free_baseline_used
+        )
+        exp = await self.engine.evaluate(
+            replace(req, commit=sha), admin=admin or free_baseline
+        )
+        # Consume the free slot only after the eval actually succeeds. Setting it
+        # before the await would burn the one free baseline on a transient engine
+        # failure (timeout, infra), forcing the agent to pay for the retry, which is
+        # the exact failure mode this feature prevents. Safe in the single-threaded
+        # asyncio loop: no await runs between the check above and this write.
+        if free_baseline:
+            self._free_baseline_used = True
+        # Route with the agent's real tier even when the eval was unmetered.
         result_path = self._route_results(exp, admin=admin)
         budget_remaining = None
         if not admin:
@@ -99,6 +126,10 @@ def status(self) -> StatusSummary:
             submit_enabled=self.submit_enabled,
             budget=self.engine.budget.status(),
             split_accesses=self.split_accesses,
+            base_commit=self.base_commit,
+            free_baseline_available=(
+                self.base_commit is not None and not self._free_baseline_used
+            ),
         )
 
     # ------------------------------------------------------------------

vero/tests/test_harbor_server.py

@@ -41,7 +41,7 @@ def _experiment(split: str, commit: str = "abcdef123456") -> Experiment:
     )
 
 
-def _sidecar(tmp_path, *, split, submit_enabled=False):
+def _sidecar(tmp_path, *, split, submit_enabled=False, base_commit=None):
     engine = MagicMock()
     engine.evaluate = AsyncMock(return_value=_experiment(split))
     engine.budget = BudgetLedger(
@@ -54,6 +54,7 @@ def _sidecar(tmp_path, *, split, submit_enabled=False):
         agent_volume=tmp_path / "agent_vol",
         admin_volume=tmp_path / "admin_vol",
         submit_enabled=submit_enabled,
+        base_commit=base_commit,
     )
     # Stub the git transfer (integration-tested separately); pin the sha.
     sidecar._transfer_commit = AsyncMock(return_value="abcdef123456")
@@ -122,3 +123,80 @@ def test_status_reports_submit_and_splits(self, tmp_path):
         assert status.submit_enabled is True
         assert status.splits[0]["split"] == "train"
         assert status.splits[0]["remaining_run_budget"] == 5
+
+
+class TestFreeBaselineEval:
+    """The agent's first eval of the seeded baseline is budget-free: it is the
+    reference every candidate is compared to and can never win selection, so
+    metering it forced a choice between optimizing blind and wasting budget
+    (observed live: exp5's optimizer skipped the reference, could not tell a
+    no-op edit from an improvement, and quit with budget unspent)."""
+
+    @pytest.mark.asyncio
+    async def test_first_baseline_eval_is_unmetered(self, tmp_path):
+        sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456")
+        await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        # engine.evaluate was called with admin=True (bypasses the ledger)
+        assert sidecar.engine.evaluate.await_args.kwargs["admin"] is True
+        # but results were routed with the agent tier (summary written)
+        dest = tmp_path / "agent_vol" / "results" / "validation__abcdef123456"
+        assert (dest / "summary.json").exists()
+
+    @pytest.mark.asyncio
+    async def test_second_baseline_eval_is_metered(self, tmp_path):
+        sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456")
+        await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        assert sidecar.engine.evaluate.await_args.kwargs["admin"] is False
+
+    @pytest.mark.asyncio
+    async def test_non_baseline_commit_always_metered(self, tmp_path):
+        sidecar = _sidecar(tmp_path, split="validation", base_commit="other000000")
+        await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        assert sidecar.engine.evaluate.await_args.kwargs["admin"] is False
+
+    @pytest.mark.asyncio
+    async def test_no_base_commit_never_free(self, tmp_path):
+        sidecar = _sidecar(tmp_path, split="validation")  # base_commit=None
+        await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        assert sidecar.engine.evaluate.await_args.kwargs["admin"] is False
+
+    def test_status_surfaces_free_baseline(self, tmp_path):
+        sidecar = _sidecar(tmp_path, split="train", base_commit="abcdef123456")
+        s = sidecar.status()
+        assert s.base_commit == "abcdef123456"
+        assert s.free_baseline_available is True
+
+    @pytest.mark.asyncio
+    async def test_status_flips_after_use(self, tmp_path):
+        sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456")
+        await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        assert sidecar.status().free_baseline_available is False
+
+    @pytest.mark.asyncio
+    async def test_admin_baseline_eval_does_not_consume_free_slot(self, tmp_path):
+        # An admin re-score of the base commit (finalize, score_baseline) must leave
+        # the agent's free slot intact: the guard is `not admin`, but pin it so a
+        # refactor that moves the flag outside the `if free_baseline` block regresses.
+        sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456")
+        await sidecar.evaluate(
+            EvalRequest(dataset_id="ds1", split="validation"), admin=True
+        )
+        assert sidecar._free_baseline_used is False
+        assert sidecar.status().free_baseline_available is True
+
+    @pytest.mark.asyncio
+    async def test_failed_baseline_eval_leaves_free_slot_available(self, tmp_path):
+        # A transient engine failure must NOT burn the one free baseline: the agent
+        # should be able to retry for free. The flag is consumed only after success.
+        sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456")
+        sidecar.engine.evaluate = AsyncMock(side_effect=RuntimeError("transient infra"))
+        with pytest.raises(RuntimeError):
+            await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        assert sidecar._free_baseline_used is False
+        assert sidecar.status().free_baseline_available is True
+        # The retry is still granted for free (admin=True bypasses the ledger).
+        sidecar.engine.evaluate = AsyncMock(return_value=_experiment("validation"))
+        await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation"))
+        assert sidecar.engine.evaluate.await_args.kwargs["admin"] is True
+        assert sidecar._free_baseline_used is True