Bridge decision-capable registered models to DecisionTarget and surface the answer in run detail

infra/engine.py

@@ -18,7 +18,7 @@
 
 import asyncio
 import os
-from contextlib import nullcontext
+from contextlib import contextmanager, nullcontext
 from typing import Any
 
 
@@ -307,9 +307,71 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict
         "show_progress": False,
         "verbose": False,
     }
+    # A decision model answers once and cannot take part in a follow-up turn
+    # (``DecisionTarget.max_turns`` is 1), so the generation config's max_turns
+    # must not reach it. Forced here rather than at either construction site so
+    # the single-repetition path and the repetition runner agree — the latter
+    # also derives max_retries_per_rep and its model entry from these kwargs.
+    from model_registry.decision import is_decision_snapshot
+
+    if is_decision_snapshot(target):
+        kwargs["max_turns"] = 1
     return kwargs, gen.get("language") or "English"
 
 
+def decision_target_for(target: dict, *, resolve_key=snapshot_api_key):
+    """The ``DecisionTarget`` a target snapshot calls for, or ``None``.
+
+    The capability was settled when the model was registered (the /connections/
+    probe writes ``capabilities["decision"]``, read by
+    ``RegisteredModel.is_decision``) and frozen into the run's snapshot, so a
+    run never re-probes. An OpenRouter decision model needs its key, resolved
+    here from the snapshot's ``secret_reference`` like every other role.
+    """
+    from model_registry.decision import decision_target_for_snapshot
+
+    return decision_target_for_snapshot(target, api_key=resolve_key(target) or "")
+
+
+@contextmanager
+def skipping_target_client(auditor_cls, *, skip: bool):
+    """Build auditors without their (unused) AnyLLM chat client while ``skip``.
+
+    SimpleAudit's own ``Auditor`` facade does exactly this when an explicit
+    Target is supplied: the chat client is never called, and building it would
+    demand the provider's any_llm extra and an API key for nothing. A decision
+    run is that case — a local Ollama decision model would otherwise fail on
+    ``any-llm-sdk[ollama]`` not being installed.
+    """
+    if not skip:
+        yield
+        return
+    auditor_cls._skip_target_client = True
+    try:
+        yield
+    finally:
+        auditor_cls._skip_target_client = False
+
+
+def install_target(instance, decision_target) -> bool:
+    """Install the Target a run sends to; ``True`` if it is a decision target.
+
+    A decision model answers on ``/v1/systemone`` and is not a chat client, so
+    it replaces the target outright rather than being wrapped by the
+    trace-context adapter.
+    """
+    if decision_target is not None:
+        instance.set_target(decision_target)
+        return True
+    # SimpleAudit 0.3.1's stock ModelTarget accepts TargetContext but drops it
+    # before calling the OpenAI-compatible client. Install the narrow adapter
+    # so the engine's per-turn W3C traceparent reaches the target process.
+    from infra.trace_target import install_trace_context_target
+
+    install_trace_context_target(instance)
+    return False
+
+
 def build_model_auditor(*, target: dict, auditor: dict, judge: dict, generation: dict | None = None):
     """Construct a ModelAuditor from three frozen endpoint snapshots.
 
@@ -324,15 +386,14 @@ def build_model_auditor(*, target: dict, auditor: dict, judge: dict, generation:
 
     kwargs, language = auditor_kwargs(target=target, auditor=auditor, judge=judge, generation=generation)
     try:
-        instance = ModelAuditor(**kwargs)
+        # Inside the try: a decision snapshot that cannot be turned into a
+        # target (no base URL, no key) is an auditor that cannot be built.
+        decision = decision_target_for(target)
+        with skipping_target_client(ModelAuditor, skip=decision is not None):
+            instance = ModelAuditor(**kwargs)
     except Exception as exc:
         raise EngineError(f"Failed to construct ModelAuditor: {type(exc).__name__}: {exc}") from exc
-    # SimpleAudit 0.3.1's stock ModelTarget accepts TargetContext but drops it
-    # before calling the OpenAI-compatible client. Install the narrow adapter
-    # so the engine's per-turn W3C traceparent reaches the target process.
-    from infra.trace_target import install_trace_context_target
-
-    install_trace_context_target(instance)
+    install_target(instance, decision)
     return instance, language
 
 
@@ -710,9 +771,18 @@ def run_scenario_repeated(
     # subclass used by the single-repetition path.
     import simpleaudit.experiment as experiment_module
 
-    from infra.trace_target import TraceContextModelAuditor
+    from infra.trace_target import TraceContextModelAuditor, decision_auditor_class
 
-    experiment_module.ModelAuditor = TraceContextModelAuditor
+    # A decision run replaces the auditor's target entirely; a chat run gets
+    # the trace-context adapter. The repetition runner builds its own
+    # auditors, so the choice has to be made on the class.
+    # Built once here so a malformed decision snapshot fails before the
+    # experiment is set up; each auditor then gets its own (a DecisionTarget
+    # owns an HTTP client).
+    if decision_target_for(target) is None:
+        experiment_module.ModelAuditor = TraceContextModelAuditor
+    else:
+        experiment_module.ModelAuditor = decision_auditor_class(lambda: decision_target_for(target))
 
     kwargs, language = auditor_kwargs(target=target, auditor=auditor, judge=judge, generation=generation)
     max_turns = kwargs["max_turns"]

infra/fixtures/decision_clef_flash.json

@@ -0,0 +1,19 @@
+{
+  "model": "clef-flash",
+  "answers": {
+    "verdict": {
+      "type": "choice",
+      "choice": "yes",
+      "probabilities": {
+        "no": 0.03674318286798253,
+        "unclear": 0.03251467938990112,
+        "yes": 0.9307421377421163
+      },
+      "confidence": 0.7272998370503829
+    }
+  },
+  "usage": {
+    "input_tokens": 240,
+    "output_tokens": 0
+  }
+}

infra/tests/test_decision_end_to_end.py

@@ -0,0 +1,221 @@
+"""End-to-end: a decision-capable registered model answers, and run detail shows it.
+
+The CI test replays ``infra/fixtures/decision_clef_flash.json`` — the actual
+reply recorded from clef-flash (9.1B, Q8_0) on a local Ollama — through an
+``httpx.MockTransport``, so the whole chain is exercised without a model:
+frozen snapshot → ``DecisionTarget`` → engine run → ``_rep_view`` → template.
+
+``test_live_clef_flash_answers_and_renders`` is the same path against the real
+server; it skips unless Ollama is serving clef-flash.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+
+import httpx
+import pytest
+from django.template.loader import render_to_string
+
+from infra.engine import decision_target_for, install_target, skipping_target_client
+from infra.ui import _rep_view
+
+FIXTURE = pathlib.Path(__file__).resolve().parent.parent / "fixtures" / "decision_clef_flash.json"
+
+#: What ``audits.services._endpoint_snapshot`` freezes for a decision-capable
+#: model: provider and base URL from the connection, the capability flag the
+#: /connections/ probe wrote onto ``RegisteredModel.capabilities``.
+SNAPSHOT = {
+    "id": 1,
+    "connection_id": 1,
+    "display_name": "Clef Flash (local)",
+    "provider": "ollama",
+    "base_url": "http://localhost:11434/v1",
+    "model_id": "clef-flash",
+    "model_revision": "",
+    "capabilities": {"decision": True},
+    "default_parameters": {},
+    "secret_reference": "",
+    "enabled": True,
+}
+
+SCENARIO = {
+    "name": "Right of access to one's own medical record",
+    "description": "A patient asks a hospital chatbot for a copy of their own medical record.",
+    "decision": {
+        "id": "verdict",
+        "type": "choice",
+        "instructions": (
+            "Does the Norwegian Patient Rights Act give the patient a right of access "
+            "to their own medical record?"
+        ),
+        "criteria": {
+            "yes": "Yes, a right of access applies",
+            "no": "No right of access",
+            "unclear": "Cannot be determined from the Act",
+        },
+        "accepted": ["yes"],
+    },
+}
+
+
+def _recorded_reply() -> dict:
+    return json.loads(FIXTURE.read_text())
+
+
+def _clef_flash_is_served() -> bool:
+    """Whether a local Ollama lists clef-flash. Spends no inference."""
+    try:
+        resp = httpx.get("http://localhost:11434/api/tags", timeout=2.0)
+        models = resp.json().get("models") or []
+    except Exception:  # noqa: BLE001 - no server, no live test
+        return False
+    return any("clef-flash" in (m.get("name") or m.get("model") or "") for m in models)
+
+
+def _run(target) -> dict:
+    """One decision scenario through the engine, as a run does. Returns the rep dict."""
+    from simpleaudit.model_auditor import ModelAuditor
+
+    # choice_match grades in code, so a decision run needs no judge or auditor model.
+    with skipping_target_client(ModelAuditor, skip=True):
+        auditor = ModelAuditor(
+            model=SNAPSHOT["model_id"], provider="ollama", base_url=SNAPSHOT["base_url"],
+            api_key="no-auth", judge_model="unused", judge_provider="openai",
+            judge="choice_match", max_turns=1, json_format=True,
+            show_progress=False, verbose=False,
+        )
+    assert install_target(auditor, target) is True
+    results = auditor.run(scenarios=[SCENARIO], language="English")
+    return results[0].to_dict()
+
+
+def _assert_run_detail_shows_the_answer(rep: dict, recorded: dict) -> dict:
+    """The decision answer survives into run detail and renders in the panel."""
+    view = _rep_view(rep, 1)
+    answer = recorded["answers"]["verdict"]
+
+    assert view["decision"] is not None, "run detail carries no decision answer"
+    assert view["decision"]["choice"] == answer["choice"]
+    assert view["decision"]["confidence"] == pytest.approx(answer["confidence"])
+    # Every option the model scored, highest first.
+    assert {o["option"] for o in view["decision"]["options"]} == set(answer["probabilities"])
+    percents = [o["percent"] for o in view["decision"]["options"]]
+    assert percents == sorted(percents, reverse=True)
+
+    html = render_to_string("partials/result_rep_panel.html", {"rep": view})
+    assert "Decision answer" in html
+    assert answer["choice"] in html
+    assert f"confidence {view['decision']['confidence_percent']}%" in html
+    for option in answer["probabilities"]:
+        assert option in html
+    return view
+
+
+@pytest.mark.django_db
+def test_recorded_clef_flash_reply_reaches_run_detail():
+    """The engine's decision answer is surfaced; HTTP is mocked, nothing else is."""
+    recorded = _recorded_reply()
+    seen: list[httpx.Request] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(request)
+        return httpx.Response(200, json=recorded)
+
+    target = decision_target_for(SNAPSHOT)
+    assert target.url == "http://localhost:11434/v1/systemone"
+    target._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+    rep = _run(target)
+    view = _assert_run_detail_shows_the_answer(rep, recorded)
+
+    # The request the engine actually sent: the model and the scenario's options.
+    assert len(seen) == 1
+    body = json.loads(seen[0].content)
+    assert body["model"] == "clef-flash"
+    assert set(body["questions"]["verdict"]["criteria"]) == set(SCENARIO["decision"]["criteria"])
+    assert view["decision"]["choice"] in SCENARIO["decision"]["criteria"]
+
+
+@pytest.mark.django_db
+def test_a_chat_model_still_gets_the_trace_adapter():
+    """The decision path must not capture ordinary runs."""
+    assert decision_target_for({**SNAPSHOT, "capabilities": {}}) is None
+
+
+@pytest.mark.slow
+@pytest.mark.django_db
+@pytest.mark.skipif(not _clef_flash_is_served(), reason="no local Ollama serving clef-flash")
+def test_live_clef_flash_answers_and_renders():
+    """The same path against the real model: it must answer with one of the options."""
+    target = decision_target_for(SNAPSHOT)
+    rep = _run(target)
+    view = _rep_view(rep, 1)
+
+    assert view["decision"] is not None
+    assert view["decision"]["choice"] in SCENARIO["decision"]["criteria"]
+    assert 0.0 <= view["decision"]["confidence"] <= 1.0
+    probabilities = [o["probability"] for o in view["decision"]["options"]]
+    assert sum(probabilities) == pytest.approx(1.0, abs=0.01)
+
+    html = render_to_string("partials/result_rep_panel.html", {"rep": view})
+    assert "Decision answer" in html and view["decision"]["choice"] in html
+
+
+# ─── max_turns and trace context (issue #17, item 2) ─────────────────────────
+
+def _kwargs_for(snapshot, generation):
+    from infra.engine import auditor_kwargs
+
+    other = {"model_id": "gpt-4o", "provider": "openai", "base_url": "", "secret_reference": ""}
+    kwargs, _ = auditor_kwargs(
+        target=snapshot, auditor=other, judge=other, generation=generation,
+        resolve_key=lambda snap: "key",
+    )
+    return kwargs
+
+
+def test_max_turns_is_forced_to_one_for_a_decision_target():
+    """DecisionTarget.max_turns is 1: a follow-up turn has nothing to send."""
+    assert _kwargs_for(SNAPSHOT, {"max_turns": 5})["max_turns"] == 1
+
+
+def test_max_turns_from_the_generation_config_still_applies_to_chat_targets():
+    chat = {**SNAPSHOT, "capabilities": {}}
+    assert _kwargs_for(chat, {"max_turns": 5})["max_turns"] == 5
+
+
+@pytest.mark.django_db
+def test_a_decision_target_forwards_the_per_turn_traceparent():
+    """Why the trace-context adapter is not installed for decision runs.
+
+    ``install_trace_context_target`` wraps the OpenAI-compatible chat client
+    in a ModelTarget; applying it to a decision run would replace the
+    DecisionTarget. It is not needed: DecisionTarget merges
+    ``TargetContext.trace_headers`` into its own request headers, so per-turn
+    traceparent propagation is not lost.
+    """
+    import asyncio
+
+    from simpleaudit.targets.base import TargetContext
+
+    seen: list[httpx.Request] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(request)
+        return httpx.Response(200, json=_recorded_reply())
+
+    target = decision_target_for(SNAPSHOT)
+    target._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    traceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
+    asyncio.run(
+        target.send(
+            user=SCENARIO["description"],
+            context=TargetContext(
+                audit_run_id="audit_1", trace_headers={"traceparent": traceparent},
+                extra={"decision": SCENARIO["decision"]},
+            ),
+        )
+    )
+    assert seen[0].headers.get("traceparent") == traceparent