diff --git a/src/leapflow/cli/context.py b/src/leapflow/cli/context.py index 31157a10..4393b191 100644 --- a/src/leapflow/cli/context.py +++ b/src/leapflow/cli/context.py @@ -2354,6 +2354,7 @@ async def _archive_to_semantic(messages: List[Dict[str, Any]]) -> None: max_consecutive_same=settings.guardrail_max_consecutive_same, argument_scoped_tools=settings.guardrail_argument_scoped_tools, max_calls_per_turn=settings.guardrail_max_calls_per_turn, + max_calls_per_turn_hard=settings.guardrail_max_calls_per_turn_hard, ) logger.debug("Tool loop guardrails enabled") else: diff --git a/src/leapflow/config.py b/src/leapflow/config.py index 9a75d248..5e12db31 100644 --- a/src/leapflow/config.py +++ b/src/leapflow/config.py @@ -757,6 +757,8 @@ class Settings: guardrail_stagnation_window: int = 10 guardrail_min_success_rate: float = 0.2 guardrail_max_calls_per_turn: int = 50 + guardrail_max_calls_per_turn_hard: int = 150 + guardrail_calls_extension_step: int = 25 # ── Session Persistence ── session_persistence_enabled: bool = True @@ -1704,6 +1706,8 @@ def _build_settings_from_env( guardrail_stagnation_window = int(os.getenv("LEAPFLOW_GUARDRAIL_STAGNATION_WINDOW", "10")) guardrail_min_success_rate = float(os.getenv("LEAPFLOW_GUARDRAIL_MIN_SUCCESS_RATE", "0.2")) guardrail_max_calls_per_turn = int(os.getenv("LEAPFLOW_GUARDRAIL_MAX_CALLS_PER_TURN", "50")) + guardrail_max_calls_per_turn_hard = int(os.getenv("LEAPFLOW_GUARDRAIL_MAX_CALLS_PER_TURN_HARD", "150")) + guardrail_calls_extension_step = int(os.getenv("LEAPFLOW_GUARDRAIL_CALLS_EXTENSION_STEP", "25")) # Session Persistence session_persistence_enabled = _bool("LEAPFLOW_SESSION_PERSISTENCE_ENABLED", "true") @@ -2229,6 +2233,8 @@ def _tuple_env(key: str, default: tuple) -> tuple: guardrail_stagnation_window=guardrail_stagnation_window, guardrail_min_success_rate=guardrail_min_success_rate, guardrail_max_calls_per_turn=guardrail_max_calls_per_turn, + guardrail_max_calls_per_turn_hard=guardrail_max_calls_per_turn_hard, + guardrail_calls_extension_step=guardrail_calls_extension_step, # Session Persistence session_persistence_enabled=session_persistence_enabled, session_resume_cache_policy=session_resume_cache_policy, diff --git a/src/leapflow/config_service.py b/src/leapflow/config_service.py index c724e1e9..b34d316b 100644 --- a/src/leapflow/config_service.py +++ b/src/leapflow/config_service.py @@ -508,7 +508,9 @@ class ConfigSnapshot: "guardrail.max_consecutive_same": "Consecutive uses of the same tool that trigger a diversify nudge (suppressed while progressing, so batch/sequential work is not penalized).", "guardrail.stagnation_window": "Window of recent genuine tool results over which the low-success-rate stagnation warning is computed.", "guardrail.min_success_rate": "Minimum tool success rate within the stagnation window before a stagnation warning is emitted.", - "guardrail.max_calls_per_turn": "Hard ceiling on total tool invocations in a single agent turn; exceeding it halts unconditionally regardless of progress.", + "guardrail.max_calls_per_turn": "Soft ceiling on total tool invocations in a single agent turn; productive windows extend it by calls_extension_step up to max_calls_per_turn_hard.", + "guardrail.max_calls_per_turn_hard": "Absolute turn tool-call ceiling; reaching it halts regardless of progress. Set equal to max_calls_per_turn to disable extensions; smaller values are clamped to the soft ceiling.", + "guardrail.calls_extension_step": "Additional tool calls granted per productive window while resource budgets allow continuation, bounded by max_calls_per_turn_hard.", "tools.ripgrep_autoinstall": "Best-effort seamless ripgrep auto-install for code_search when missing (macOS/Homebrew, no sudo, background, non-fatal). code_search always works via the pure-Python fallback regardless; disabling this just skips the accelerator install and shows a manual hint.", "tools.test_command": "Explicit command for the test_run tool (empty => auto-detect pytest/npm/go/cargo from project markers).", "tools.lint_command": "Explicit command for the lint_check tool (empty => auto-detect ruff/eslint/go vet/clippy from project markers).", diff --git a/src/leapflow/engine/engine.py b/src/leapflow/engine/engine.py index 02481733..e2b75864 100644 --- a/src/leapflow/engine/engine.py +++ b/src/leapflow/engine/engine.py @@ -6,7 +6,8 @@ wrappers. Behavior is progressively extracted into delegate components and sub-packages; the remaining length is loop-iteration state shared across those components, which resists further splitting without indirection that -costs more than it saves.""" +costs more than it saves. Guardrail stop context is handed from that loop to +its existing tool-free finalization helpers; cap policy stays in tools/.""" from __future__ import annotations @@ -246,6 +247,9 @@ def __init__( # Tool loop guardrails (injected by CLI) self._guardrail: Optional[Any] = None + # The guardrail violation that stopped the current turn, if any; read by + # the forced-finalize path so the user is told why and how to resume. + self._last_guardrail_halt: Optional[Any] = None # Optional override for dynamic tool result budget (set by CLI wiring) self._tool_result_budget: Optional[int] = None @@ -2362,6 +2366,7 @@ async def _run_agent_loop( break if self._tool_dispatch._check_guardrail(messages) == "halt": + content = "" # A text tool request is not a final answer. break if status == BudgetStatus.SOFT_LIMIT and not self._should_extend_budget( @@ -2417,7 +2422,10 @@ async def _run_agent_loop( fallback = ( _app_onboarding_recovery_message(messages) or _last_tool_failures_recovery_message(messages, include_generic=False) - or await self._synthesize_forced_answer(messages) + or await self._synthesize_forced_answer( + messages, stop_reason=getattr(self._last_guardrail_halt, "reason", None) + ) + or (self._budget_exhausted_response(messages) if self._last_guardrail_halt else "") or _last_tool_failures_recovery_message(messages) or fatal_error or self._budget_exhausted_response(messages) @@ -2533,7 +2541,9 @@ async def _handle_llm_recovery( # ── Helpers ────────────────────────────────────────────────────────── - async def _synthesize_forced_answer(self, messages: List[Dict[str, Any]]) -> str: + async def _synthesize_forced_answer( + self, messages: List[Dict[str, Any]], *, stop_reason: str | None = None + ) -> str: """One tool-free LLM round to answer with what the turn already gathered. A turn can stop before the model has written a final answer: a detected @@ -2549,7 +2559,17 @@ async def _synthesize_forced_answer(self, messages: List[Dict[str, Any]]) -> str """ try: prompt = list(messages) - prompt.append(build_user_message_text(_FORCED_FINALIZE_PROMPT)) + finalize_prompt = _FORCED_FINALIZE_PROMPT + if stop_reason: + finalize_prompt += ( + f"\nThis turn stopped because: {stop_reason}. " + "In the user's language, distinguish completed steps from steps not yet executed. " + "Explain that replying 'continue' resumes the remaining work in a new turn. " + "Do not claim the task is complete without evidence. " + "Completed or uncertain side effects must be verified before repeating actions; " + "continuation does not bypass existing safety or approval gates." + ) + prompt.append(build_user_message_text(finalize_prompt)) compressed = self._prompt_assembler._prepare_llm_messages( self._healer.heal(prompt), tools=None, round_number=0 ) @@ -2571,9 +2591,17 @@ def _budget_exhausted_response(self, messages: List[Dict[str, Any]]) -> str: open-question count and next step so the stop is informative and continuable (not a bare dead-stop); otherwise the plain notice. """ - base = ( - "I've reached my reasoning step limit. Here's my best answer based on progress so far." - ) + halt = getattr(self, "_last_guardrail_halt", None) + if halt is not None: + base = ( + f"This turn stopped: {halt.reason}. " + "Reply 'continue' to continue the remaining work in a new turn. " + "Completed or uncertain actions must be verified before they are repeated." + ) + else: + base = ( + "I've reached my reasoning step limit. Here's my best answer based on progress so far." + ) led = self._research_ledger if led.is_empty or led.open_question_count == 0: return base diff --git a/src/leapflow/engine/prompt_assembler.py b/src/leapflow/engine/prompt_assembler.py index e9d046ea..943f9f96 100644 --- a/src/leapflow/engine/prompt_assembler.py +++ b/src/leapflow/engine/prompt_assembler.py @@ -80,6 +80,7 @@ def _begin_turn_context(self, user_text: str) -> None: # reflect calls made in THIS turn, not the full session. if self._engine._guardrail is not None: self._engine._guardrail.reset() + self._engine._last_guardrail_halt = None self._engine._current_command_id = self._engine._current_task_contract.task_id self._engine._tool_execution_ledger.reset(store=self._engine._conversation_store) try: diff --git a/src/leapflow/engine/session/session_factory.py b/src/leapflow/engine/session/session_factory.py index f78a292d..cba1667c 100644 --- a/src/leapflow/engine/session/session_factory.py +++ b/src/leapflow/engine/session/session_factory.py @@ -409,6 +409,16 @@ def build_session_engine( engine._usage_tracker = TurnUsageTracker() _wire_plugin_stats_sink(engine._usage_tracker) engine._recovery_coordinator = RecoveryCoordinator() + # Guards hold per-turn baselines; a shared instance lets one session's turn + # start reset the tool-call count another session is enforcing. + _base_guardrail = getattr(base_engine, "_guardrail", None) + if _base_guardrail is not None: + if callable(getattr(_base_guardrail, "fresh", None)): + engine._guardrail = _base_guardrail.fresh() + else: + engine._guardrail = copy.deepcopy(_base_guardrail) + engine._guardrail.reset() + engine._last_guardrail_halt = None engine._last_context_snapshot = {} engine._last_turn_tool_categories = frozenset() # Clean per-turn state slate (each turn also reassigns these, but a fresh diff --git a/src/leapflow/engine/tool_dispatch_engine.py b/src/leapflow/engine/tool_dispatch_engine.py index f2bc0d51..43a22880 100644 --- a/src/leapflow/engine/tool_dispatch_engine.py +++ b/src/leapflow/engine/tool_dispatch_engine.py @@ -9,7 +9,8 @@ Size note (>800-line registered exception per AGENTS.md): dispatch, catalog assembly, side-effect batch gating, and guardrail evaluation share one dispatch state machine; splitting them would fork the failure path the -gating exists to protect. +gating exists to protect. Productive turn-cap extension remains part of this +shared evaluation path, before its terminal verdict is handed to the loop. """ from __future__ import annotations @@ -98,9 +99,12 @@ def _check_guardrail( # trip, leaving a genuine no-op loop to spin until the budget is spent. frame = self._engine._active_frame stalled = bool(frame is not None and getattr(frame, "stalled_rounds", 0) >= 1) + if violation.severity == "halt" and self._try_extend_turn_cap(violation): + return None if violation.severity == "halt" and ( getattr(violation, "progress_independent", False) or stalled ): + self._engine._last_guardrail_halt = violation messages.append( build_user_message_text( f"SYSTEM GUARDRAIL: {violation.reason}. {violation.suggestion}" @@ -113,6 +117,28 @@ def _check_guardrail( build_user_message_text(f"SYSTEM WARNING: {violation.reason}. {violation.suggestion}") ) return None + + def _try_extend_turn_cap(self, violation: Any) -> bool: + """Lift a productive soft turn-cap halt while resource budgets allow it. + + Nothing is appended to the conversation: the extension is invisible to + the model, so the cached prompt prefix is unaffected. + """ + if not getattr(violation, "extendable", False): + return False + if not self._engine._within_resource_limits(): + return False + extend = getattr(self._engine._guardrail, "extend_turn_cap", None) + if not callable(extend): + return False + settings = getattr(self._engine, "_settings", None) + step = int(getattr(settings, "guardrail_calls_extension_step", 25) or 25) + new_cap = extend(step) + if new_cap is None: + return False + logger.info("guardrail: turn tool-call cap extended to %d (progress-gated)", new_cap) + return True + def _evaluate_tool_failures( self, failed_items: List[tuple[str, Dict[str, Any]]], diff --git a/src/leapflow/engine/tools/tool_guardrails.py b/src/leapflow/engine/tools/tool_guardrails.py index df4a9b24..0e10c33a 100644 --- a/src/leapflow/engine/tools/tool_guardrails.py +++ b/src/leapflow/engine/tools/tool_guardrails.py @@ -15,6 +15,7 @@ """ from __future__ import annotations +import copy import hashlib import json import logging @@ -22,7 +23,12 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Protocol, runtime_checkable -from leapflow.engine.tools.tool_execution import exit_code_from +from leapflow.engine.tools.tool_execution import ( + TIER_REPAIRED, + TIER_SUCCEEDED, + exit_code_from, + result_tier, +) logger = logging.getLogger(__name__) @@ -116,11 +122,14 @@ class GuardrailViolation: severity: str = "warning" # "warning" | "halt" suggestion: str = "" # A halt the engine must honour regardless of the global stall marker. - # Set only when the violation is, by construction, zero forward progress - # (e.g. the same tool returned the same result N times), so that the coarse - # research/governance progress heuristic cannot keep a genuine no-op loop - # spinning until the iteration budget is exhausted. + # Used for definitionally zero progress (identical call/result loops) and + # resource ceilings. A productive soft ceiling can still be extended, but + # the coarse research/governance marker cannot suppress a terminal verdict. progress_independent: bool = False + # The halt may be lifted by raising a soft ceiling, because the window + # that reached it was productive. The engine grants the extension only + # when real resource budgets also allow it; otherwise the halt stands. + extendable: bool = False @runtime_checkable @@ -348,7 +357,7 @@ def reset(self) -> None: class TurnCapGuard: - """Enforce a hard ceiling on tool invocations within a single agent turn. + """Bound tool invocations within a single agent turn: soft cap, hard cap. Counts only the tool calls added *during the current turn*, not calls from prior turns that may be present in the conversation history. A deferred @@ -360,37 +369,111 @@ class TurnCapGuard: on the initial message list before any tool calls are executed. The resulting halt is ``progress_independent`` because an unbounded turn is a resource hazard regardless of whether the task is making headway. + + Reaching the soft cap (``max_calls``) yields an ``extendable`` halt when + the calls since the last checkpoint were productive -- mostly successful + and mostly distinct -- so a long multi-step task (e.g. a physical + manipulation sequence) is not cut off mid-way while a spinning one still + stops. ``hard_max_calls`` is never extended; equal to ``max_calls`` (the + default) it restores the flat-cap behaviour. """ - def __init__(self, *, max_calls: int = 50) -> None: + def __init__( + self, + *, + max_calls: int = 50, + hard_max_calls: Optional[int] = None, + min_window_success: float = 0.5, + min_window_distinct: float = 0.5, + ) -> None: self._max_calls = max_calls + self._hard_max_calls = max(max_calls, hard_max_calls or max_calls) + self._min_window_success = min_window_success + self._min_window_distinct = min_window_distinct + self._cap = max_calls # Deferred baseline: set on the first check() after reset() to the # number of pre-existing tool calls in the conversation history. self._baseline: Optional[int] = None + # Absolute call index where the window judged at the next cap starts. + self._window_start: Optional[int] = None + self._last_total = 0 + + @property + def cap(self) -> int: + """The ceiling currently in force for this turn.""" + return self._cap @staticmethod - def _count_calls(history: List[Dict[str, Any]]) -> int: - return sum( - len(msg.get("tool_calls") or []) - for msg in history - if msg.get("role") == "assistant" - ) + def _call_records( + history: List[Dict[str, Any]], + ) -> tuple[List[tuple[str, str, Any]], Dict[str, Any]]: + """Correlate native records and text-mode call/result pairs without mutation.""" + from leapflow.skills.tool_call_parser import parse_tool_call + + calls: List[tuple[str, str, Any]] = [] + results: Dict[str, Any] = {} + pending_text_id: str | None = None + for index, msg in enumerate(history): + content = msg.get("content", "") + if msg.get("role") == "assistant": + pending_text_id = None + native_calls = msg.get("tool_calls") or [] + for tc in native_calls: + fn = tc.get("function", {}) or {} + raw = fn.get("arguments", "{}") + try: + args = json.loads(raw) if isinstance(raw, str) else raw + except (TypeError, ValueError): + args = raw + calls.append((str(tc.get("id", "")), str(fn.get("name", "")), args)) + if not native_calls and isinstance(content, str): + text_call = parse_tool_call(content) + if text_call is not None: + pending_text_id = f"text-history-{index}" + calls.append((pending_text_id, text_call.name, text_call.params)) + continue + if msg.get("role") == "tool": + call_id = str(msg.get("tool_call_id", "")) + elif pending_text_id is not None and StagnationGuard._is_tool_result_message(msg): + call_id = pending_text_id + content = content.partition("\n")[2] + pending_text_id = None + else: + continue + try: + results[call_id] = json.loads(content) if isinstance(content, str) else content + except (TypeError, ValueError): + results[call_id] = content + return calls, results + + @classmethod + def _count_calls(cls, history: List[Dict[str, Any]]) -> int: + return len(cls._call_records(history)[0]) def check(self, history: List[Dict[str, Any]]) -> GuardrailViolation: total = self._count_calls(history) + self._last_total = total if self._baseline is None: # First check this turn: snapshot the count of pre-existing calls # so only calls added after this point count towards the cap. self._baseline = total + self._window_start = total return GuardrailViolation(violated=False) calls_this_turn = total - self._baseline - if calls_this_turn >= self._max_calls: + if calls_this_turn >= self._cap: + extendable = ( + calls_this_turn < self._hard_max_calls + and self._cap < self._hard_max_calls + and self._window_productive(history) + ) + limit = self._hard_max_calls if calls_this_turn >= self._hard_max_calls else self._cap return GuardrailViolation( violated=True, severity="halt", progress_independent=True, + extendable=extendable, reason=( - f"Turn tool-call cap reached ({calls_this_turn}/{self._max_calls})" + f"Turn tool-call cap reached ({calls_this_turn}/{limit})" ), suggestion=( "Provide the best answer with the information gathered so far." @@ -398,9 +481,45 @@ def check(self, history: List[Dict[str, Any]]) -> GuardrailViolation: ) return GuardrailViolation(violated=False) + def extend(self, step: int) -> int: + """Raise the current cap by ``step`` (bounded by the hard cap). + + The calls made so far become the closed window; the next cap is judged + only on calls made after this point, so one productive stretch cannot + carry a later spinning one. + """ + reached = max(self._cap, self._last_total - (self._baseline or 0)) + self._cap = min(self._hard_max_calls, reached + max(1, int(step))) + self._window_start = self._last_total + return self._cap + + def _window_productive(self, history: List[Dict[str, Any]]) -> bool: + """Whether the calls since the last checkpoint mostly succeeded and differed.""" + calls, results = self._call_records(history) + window = calls[self._window_start or 0:] + if not window: + return False + succeeded = sum( + 1 for call_id, _, _ in window + if call_id in results + and not (isinstance(results[call_id], dict) and results[call_id].get("ok") is False) + and result_tier(results[call_id]) in (TIER_SUCCEEDED, TIER_REPAIRED) + ) + distinct = len({ + (name, json.dumps(args, sort_keys=True, default=str, separators=(",", ":"))) + for _, name, args in window + }) + return ( + succeeded / len(window) >= self._min_window_success + and distinct / len(window) >= self._min_window_distinct + ) + def reset(self) -> None: - """Clear the per-turn baseline so the next check() re-snapshots.""" + """Clear the per-turn baseline and restore the soft cap.""" self._baseline = None + self._window_start = None + self._last_total = 0 + self._cap = self._max_calls class CompositeGuardrail: @@ -416,14 +535,34 @@ def __init__( max_consecutive_same: int = 5, argument_scoped_tools: tuple[str, ...] = (), max_calls_per_turn: int = 50, + max_calls_per_turn_hard: Optional[int] = None, ) -> None: self._guards: List[ToolLoopGuard] = guards or [ RepetitionGuard(max_repeats=max_repeats), StagnationGuard(window=stagnation_window, min_success_rate=min_success_rate), DominationGuard(max_consecutive_same=max_consecutive_same, argument_scoped_tools=argument_scoped_tools), - TurnCapGuard(max_calls=max_calls_per_turn), + TurnCapGuard(max_calls=max_calls_per_turn, hard_max_calls=max_calls_per_turn_hard), ] + def fresh(self) -> "CompositeGuardrail": + """Return an independent copy: same configuration, cleared state. + + Guards carry per-turn state (e.g. ``TurnCapGuard``'s baseline), so each + session engine needs its own instance; sharing one lets a turn starting + in one session reset the baseline another session is counting against. + """ + guards: List[ToolLoopGuard] = [copy.deepcopy(guard) for guard in self._guards] + for guard in guards: + guard.reset() + return CompositeGuardrail(guards=guards) + + def extend_turn_cap(self, step: int) -> Optional[int]: + """Raise the turn tool-call cap; returns the new cap, or None if absent.""" + for guard in self._guards: + if isinstance(guard, TurnCapGuard): + return guard.extend(step) + return None + def check(self, history: List[Dict[str, Any]]) -> GuardrailViolation: worst: Optional[GuardrailViolation] = None for guard in self._guards: diff --git a/src/leapflow/version.py b/src/leapflow/version.py index 8b1a415d..7b37d912 100644 --- a/src/leapflow/version.py +++ b/src/leapflow/version.py @@ -1,4 +1,4 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Version information for leapflow.""" -__version__ = "0.8.2+main" +__version__ = "0.8.3+main" diff --git a/tests/test_agent_execution.py b/tests/test_agent_execution.py index fab445a8..73423e4f 100644 --- a/tests/test_agent_execution.py +++ b/tests/test_agent_execution.py @@ -1014,6 +1014,230 @@ def _assistant_with_n_calls(n: int, start_id: int = 0) -> list: assert v.progress_independent +def _turn_cap_history(results: list, *, start: int = 0, distinct: bool = True) -> list[dict]: + """Build correlated native assistant/tool records for a cap evaluation window.""" + import json + + history = [] + for index, result in enumerate(results, start): + history.append({ + "role": "assistant", + "tool_calls": [{"id": str(index), "function": { + "name": "hw_skill_run", "arguments": {"step": index if distinct else 0}, + }}], + }) + if result is not None: + history.append({"role": "tool", "tool_call_id": str(index), "content": json.dumps(result)}) + return history + + +def test_turn_cap_extension_uses_only_current_window_and_resets() -> None: + from leapflow.engine.tools.tool_guardrails import TurnCapGuard + + guard = TurnCapGuard(max_calls=4, hard_max_calls=8) + history = _turn_cap_history([{"ok": True}] * 120) + assert not guard.check(history).violated + history += _turn_cap_history([{"ok": True}] * 4, start=120) + assert guard.check(history).extendable + assert guard.extend(2) == 6 + history += _turn_cap_history([{"ok": False}] * 2, start=124) + violation = guard.check(history) + assert violation.violated and not violation.extendable + guard.reset() + assert guard.cap == 4 + assert not guard.check(history).violated + history += _turn_cap_history([{"ok": True}] * 4, start=126) + assert guard.check(history).extendable + assert guard.extend(100) == 8 + history += _turn_cap_history([{"ok": True}] * 4, start=130) + assert not guard.check(history).extendable # Absolute ceiling. + + +@pytest.mark.parametrize("results,distinct,extendable", [ + ([{"ok": True}] * 4, True, True), + ([{"ok": True}] * 2 + [{"ok": False}] * 2, True, True), + ([{"ok": True}] + [{"ok": False}] * 3, True, False), + ([{"ok": True}] * 4, False, False), + ([None] * 4, True, False), + ([{"ok": False, "counts_as_failure": False}] * 4, True, False), + ([{"ok": False, "failure_class": "invalid_request"}] * 4, True, False), + ([{"ok": False, "side_effect_state": "none"}] * 4, True, False), + ([{"ok": False, "duplicate_suppressed": True}] * 4, True, False), + ([{"ok": True, "argument_repair": {"field": "x"}}] * 4, True, True), +]) +def test_turn_cap_window_productivity(results, distinct, extendable) -> None: + from leapflow.engine.tools.tool_guardrails import TurnCapGuard + + guard = TurnCapGuard(max_calls=4, hard_max_calls=8) + guard.check([]) + violation = guard.check(_turn_cap_history(results, distinct=distinct)) + assert violation.violated and violation.progress_independent + assert violation.extendable is extendable + + +@pytest.mark.parametrize("hard_cap,count", [(None, 4), (4, 4), (2, 4), (6, 6), (6, 9)]) +def test_turn_cap_absolute_ceiling_including_batch_overshoot(hard_cap, count) -> None: + from leapflow.engine.tools.tool_guardrails import TurnCapGuard + + guard = TurnCapGuard(max_calls=4, hard_max_calls=hard_cap) + guard.check([]) + violation = guard.check(_turn_cap_history([{"ok": True}] * count)) + assert violation.violated and not violation.extendable + expected_limit = max(4, hard_cap or 4) + assert violation.reason.endswith(f"({count}/{expected_limit})") + + +def test_turn_cap_canonical_arguments_and_configurable_ratios() -> None: + from leapflow.engine.tools.tool_guardrails import TurnCapGuard + + history = _turn_cap_history([{"ok": True}] * 4) + for index, msg in enumerate(history[::2]): + msg["tool_calls"][0]["function"]["arguments"] = ( + '{"a": 1, "b": 2}' if index % 2 else '{"b": 2, "a": 1}' + ) + guard = TurnCapGuard(max_calls=4, hard_max_calls=8, min_window_distinct=0.5) + guard.check([]) + assert not guard.check(history).extendable # JSON key order is not progress. + guard = TurnCapGuard(max_calls=4, hard_max_calls=8, min_window_distinct=0.25) + guard.check([]) + assert guard.check(history).extendable + guard = TurnCapGuard(max_calls=4, hard_max_calls=8, min_window_success=0.75) + guard.check([]) + assert not guard.check(_turn_cap_history([{"ok": True}] * 2 + [{"ok": False}] * 2)).extendable + + +@pytest.mark.parametrize("within_limits", [True, False]) +def test_dispatch_turn_cap_extension_obeys_resources_not_coarse_stall(within_limits) -> None: + from types import SimpleNamespace + from leapflow.engine.tools.tool_guardrails import CompositeGuardrail, TurnCapGuard + + cap = TurnCapGuard(max_calls=2, hard_max_calls=6) + engine = SimpleNamespace( + _guardrail=CompositeGuardrail(guards=[cap]), + _active_frame=SimpleNamespace(stalled_rounds=8), + _settings=SimpleNamespace(guardrail_calls_extension_step=3), + _within_resource_limits=lambda: within_limits, + _last_guardrail_halt=None, + ) + engine._guardrail.check([]) + messages = _turn_cap_history([{"ok": True}] * 2) + original = list(messages) + verdict = ToolDispatchEngine(engine)._check_guardrail(messages) + if within_limits: + assert verdict is None and cap.cap == 5 + assert messages == original # No prompt/cache mutation. + assert engine._last_guardrail_halt is None + else: + assert verdict == "halt" and cap.cap == 2 + assert engine._last_guardrail_halt.reason.endswith("(2/2)") + + +def test_repetition_halt_cannot_be_lifted_by_productive_turn_cap() -> None: + from leapflow.engine.tools.tool_guardrails import CompositeGuardrail + + guard = CompositeGuardrail(max_calls_per_turn=6, max_calls_per_turn_hard=12) + guard.check([]) + history = _turn_cap_history([{"ok": True}] * 3) + history += _turn_cap_history([{"ok": True}] * 3, start=3, distinct=False) + violation = guard.check(history) + assert violation.violated and "same result" in violation.reason + assert not violation.extendable + + +def test_guardrail_finalize_prompt_and_fallback_are_actionable(monkeypatch) -> None: + from leapflow.engine.tools.tool_guardrails import GuardrailViolation + from leapflow.llm.base import LLMChatResponse + + with tempfile.TemporaryDirectory() as td: + engine, lt, _ = _adaptive_engine(td) + try: + reason = "Turn tool-call cap reached (150/150)" + engine._last_guardrail_halt = GuardrailViolation(True, reason=reason, severity="halt") + engine._llm = AsyncMock() + engine._llm.achat.return_value = LLMChatResponse(content="Remaining work needs another turn.") + monkeypatch.setattr(engine._prompt_assembler, "_prepare_llm_messages", lambda msgs, **kw: msgs) + messages = [{"role": "user", "content": "Complete the task"}] + original = list(messages) + assert asyncio.run(engine._synthesize_forced_answer(messages, stop_reason=reason)) + call = engine._llm.achat.call_args + prompt = str(call.args[0][-1]["content"]) + assert reason in prompt and "user's language" in prompt + assert "completed steps" in prompt and "not yet executed" in prompt + assert "continue" in prompt and "approval gates" in prompt + assert "tools" not in call.kwargs and messages == original + fallback = engine._budget_exhausted_response(messages) + assert reason in fallback and "Reply 'continue'" in fallback + assert "reasoning step limit" not in fallback + engine._prompt_assembler._begin_turn_context("continue") + assert engine._last_guardrail_halt is None + assert "reasoning step limit" in engine._budget_exhausted_response(messages) + finally: + lt.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("native", [True, False]) +@pytest.mark.parametrize("streaming", [True, False]) +@pytest.mark.parametrize("synthesis_fails", [True, False]) +async def test_turn_cap_extension_and_halt_through_unified_loop(native, streaming, synthesis_fails) -> None: + from dataclasses import replace + from leapflow.engine.tools.tool_guardrails import CompositeGuardrail + from leapflow.llm.base import LLMChatResponse, ToolCallInfo + + class CapLLM(StubLLM): + def __init__(self) -> None: + super().__init__() + self.prompts = [] + + async def achat(self, messages, **kwargs): + self._call_count += 1 + self.prompts.append((list(messages), kwargs)) + if self._call_count <= 4: + args = {"step": self._call_count} + if native: + return LLMChatResponse(content="", tool_calls=[ + ToolCallInfo(id=str(self._call_count), name="time_get", arguments=args), + ]) + import json + return LLMChatResponse(content='' + json.dumps({ + "name": "time_get", "arguments": args, + }) + '') + if synthesis_fails: + raise RuntimeError("final synthesis unavailable") + return LLMChatResponse(content="Partially complete. Reply 'continue' for the remaining work.") + + async def achat_stream(self, messages, **kwargs): + response = await self.achat(messages, **kwargs) + yield response.content + + with tempfile.TemporaryDirectory() as td: + engine, lt, _ = _adaptive_engine(td) + try: + engine._settings = replace( + engine._settings, native_tool_calling_enabled=native, + guardrail_calls_extension_step=2, llm_context_length=1_000_000, + ) + engine._guardrail = CompositeGuardrail(max_calls_per_turn=2, max_calls_per_turn_hard=4) + llm = CapLLM() + engine._llm = llm + if streaming: + events = [event async for event in engine.run_stream("Collect several distinct readings")] + answer = events[-1].content + assert events[-1].type == "final" + assert len([event for event in events if event.type == "tool_complete"]) == 4 + else: + answer = await engine._unified_tool_loop("Collect several distinct readings") + assert llm.call_count == 5 # Four real calls, then one tool-free finalization. + assert engine._last_guardrail_halt.reason.endswith("(4/4)") + assert "continue" in answer + if synthesis_fails: + assert "(4/4)" in answer and "reasoning step limit" not in answer + assert "(4/4)" in str(llm.prompts[-1][0]) + assert "tools" not in llm.prompts[-1][1] + finally: + lt.close() + + def test_synthesize_forced_answer_returns_model_answer() -> None: """When the loop stops without a written answer, a single tool-free round lets the model answer from the gathered context instead of emitting the canned diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py index 8aad21c3..0fc6665e 100644 --- a/tests/test_config_loader.py +++ b/tests/test_config_loader.py @@ -123,6 +123,44 @@ def test_daemon_log_level_config_chain(monkeypatch, tmp_path) -> None: assert built.daemon_log_level == "DEBUG" +def test_turn_cap_config_chain(monkeypatch, tmp_path) -> None: + """Defaults, YAML flattening, env overrides and the catalog agree on new keys.""" + from conftest import make_settings + from leapflow.config import _build_settings_from_env + from leapflow.config_service import _build_field_specs + + defaults = make_settings(str(tmp_path)) + assert defaults.guardrail_max_calls_per_turn == 50 + assert defaults.guardrail_max_calls_per_turn_hard == 150 + assert defaults.guardrail_calls_extension_step == 25 + specs = _build_field_specs() + values = { + "max_calls_per_turn": 60, + "max_calls_per_turn_hard": 180, + "calls_extension_step": 30, + } + monkeypatch.setenv("LEAPFLOW_DATA_DIR", str(tmp_path / "home")) + layout = build_layout(tmp_path / "home") + profile = layout.ensure(profile_id="default") + import yaml + profile.runtime_config_path.write_text(yaml.safe_dump({"guardrail": values}), encoding="utf-8") + for name in values: + monkeypatch.delenv(f"LEAPFLOW_GUARDRAIL_{name.upper()}", raising=False) + bundle = load_config_bundle(layout, profile, tmp_path) + for name, value in values.items(): + spec = specs[f"guardrail.{name}"] + assert spec.setting_name == f"guardrail_{name}" + assert spec.description + env = f"LEAPFLOW_GUARDRAIL_{name.upper()}" + assert bundle.env[env] == str(value) + monkeypatch.setenv(env, bundle.env[env]) + settings = _build_settings_from_env() + for name, value in values.items(): + assert getattr(settings, f"guardrail_{name}") == value + monkeypatch.setenv("LEAPFLOW_GUARDRAIL_MAX_CALLS_PER_TURN_HARD", "200") + assert _build_settings_from_env().guardrail_max_calls_per_turn_hard == 200 + + def test_hardware_alert_policy_config_chain(monkeypatch, tmp_path) -> None: from leapflow.config import _build_settings_from_env from leapflow.config_service import _build_field_specs diff --git a/tests/test_config_service.py b/tests/test_config_service.py index 7ce0e31d..8a894ee0 100644 --- a/tests/test_config_service.py +++ b/tests/test_config_service.py @@ -107,6 +107,22 @@ def test_health_and_retry_config_persists_in_correct_section(config_env, key, va assert yaml.safe_load(mutation.path.read_text())[section][name] == float(value) +@pytest.mark.parametrize("key,value", [ + ("guardrail.max_calls_per_turn", "50"), + ("guardrail.max_calls_per_turn_hard", "175"), + ("guardrail.calls_extension_step", "30"), +]) +def test_turn_cap_config_is_discoverable_and_persists(config_env, key, value) -> None: + _, service = config_env + spec = service.describe(key) + assert spec.key == key and spec.value_type == "int" + assert spec.description + mutation = service.set(key, value) + assert mutation.ok and key in mutation.changed_keys + name = key.split(".", 1)[1] + assert yaml.safe_load(mutation.path.read_text())["guardrail"][name] == int(value) + + class TestCoerceValue: """Test type coercion for config values.""" diff --git a/tests/test_session_factory.py b/tests/test_session_factory.py index 3ba8de06..d5aa6cb8 100644 --- a/tests/test_session_factory.py +++ b/tests/test_session_factory.py @@ -141,6 +141,93 @@ async def test_concurrent_session_engines_are_isolated() -> None: lt.close() +@pytest.mark.asyncio +@pytest.mark.parametrize("custom_guard", [False, True]) +async def test_concurrent_session_guardrail_baselines_are_independent(custom_guard) -> None: + """Starting B in another workspace cannot reset A's active-turn call budget.""" + from leapflow.engine.session.session_factory import build_session_engine + from leapflow.engine.tools.tool_guardrails import CompositeGuardrail, TurnCapGuard + from leapflow.memory import WorkingMemoryProvider + + class CustomGuard: + def __init__(self) -> None: + self.cap = TurnCapGuard(max_calls=3, hard_max_calls=5) + + def check(self, history): + return self.cap.check(history) + + def reset(self): + self.cap.reset() + + def history(count: int) -> list[dict]: + return [{"role": "assistant", "tool_calls": [ + {"id": str(index), "function": {"name": "read", "arguments": {"index": index}}}, + ]} for index in range(count)] + + with tempfile.TemporaryDirectory() as td: + base, lt = _build_base_engine(td, _EchoLLM()) + try: + base._guardrail = CustomGuard() if custom_guard else CompositeGuardrail( + guards=[TurnCapGuard(max_calls=3, hard_max_calls=5)], + ) + base._guardrail.check(history(120)) + base._last_guardrail_halt = object() + a, b = [build_session_engine( + base, session_id=name, working_memory=WorkingMemoryProvider(max_tokens=512), + workspace_root=f"/tmp/work-{name}", + ) for name in ("a", "b")] + assert a._guardrail is not b._guardrail and a._guardrail is not base._guardrail + assert a._last_guardrail_halt is b._last_guardrail_halt is None + a_started, b_started = asyncio.Event(), asyncio.Event() + + async def run_a(): + assert not a._guardrail.check(history(120)).violated # Fresh baseline. + a_started.set() + await asyncio.wait_for(b_started.wait(), timeout=2) + violation = a._guardrail.check(history(123)) + assert violation.violated and violation.reason.endswith("(3/3)") + assert a._current_session_id == "a" + + async def run_b(): + await asyncio.wait_for(a_started.wait(), timeout=2) + b._guardrail.reset() + assert not b._guardrail.check(history(200)).violated + b_started.set() + await asyncio.sleep(0) + assert not b._guardrail.check(history(201)).violated + assert b._current_session_id == "b" + + await asyncio.gather(run_a(), run_b()) + assert a._settings.workspace_root != b._settings.workspace_root + assert not base._guardrail.check(history(122)).violated + finally: + lt.close() + + +def test_fresh_composite_guardrail_restores_soft_cap_without_mutating_original() -> None: + from leapflow.engine.tools.tool_guardrails import CompositeGuardrail, TurnCapGuard + + cap = TurnCapGuard(max_calls=3, hard_max_calls=8) + guard = CompositeGuardrail(guards=[cap]) + guard.check([]) + assert guard.extend_turn_cap(2) == 5 + fresh = guard.fresh() + assert fresh._guards[0] is not cap + assert fresh._guards[0].cap == 3 and cap.cap == 5 + assert not fresh.check([]).violated + assert fresh.extend_turn_cap(3) == 6 and cap.cap == 5 + assert CompositeGuardrail(guards=[CustomNoCapGuard()]).extend_turn_cap(3) is None + + +class CustomNoCapGuard: + def check(self, history): + from leapflow.engine.tools.tool_guardrails import GuardrailViolation + return GuardrailViolation(violated=False) + + def reset(self): + pass + + def test_ensure_session_creates_a_provided_session_id() -> None: """A daemon-bound (client-owned) session id is created-if-not-exists, so a distinct per-TUI session persists even though the engine did not mint it."""