diff --git a/README.md b/README.md index 78b519f4..7686a380 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ The host sends **goals**; the node decomposes them locally. A skill is planned o | Act | `hw_configure` · `hw_actuate` · `hw_dispense` · `hw_batch_actuate` | approval-gated | | Safety | `hw_estop` | no approval — node-local; holds in place, reports `stop_effect` / `stop_verified` | | Control | `hw_control_start` · `hw_control_stop` · `hw_control_status` | leased deadman | -| Fleet | `hw_fleet_status` · `hw_fleet_discover` · `hw_fleet_halt_all` | multi-node | +| Fleet | `hw_fleet_status` · `hw_fleet_discover` · `hw_fleet_reconnect` · `hw_fleet_halt_all` | multi-node; unreachable nodes auto-retry | | Skills | `hw_skill_list` · `hw_skill_run` · `hw_skill_status` · `hw_skill_stop` | plan-digest approval, node-local | | Data | `hw_skill_label` | labels a recorded episode's outcome | diff --git a/src/leapflow/cli/cli.py b/src/leapflow/cli/cli.py index fa79d380..ff967c26 100644 --- a/src/leapflow/cli/cli.py +++ b/src/leapflow/cli/cli.py @@ -378,6 +378,13 @@ def main(argv: list[str] | None = None) -> int: ) hw_validate.add_argument("--devices", default="", metavar="DIR", help="Device YAML for --workcells") hw_sub.add_parser("permit-key", parents=[hw_json], help="Generate an Ed25519 keypair for execution permits") + hw_fleet = hw_sub.add_parser("fleet", help="Inspect and supervise configured LHP fleet nodes (via leapd)") + hw_fleet_sub = hw_fleet.add_subparsers(dest="fleet_action") + hw_fleet_reconnect = hw_fleet_sub.add_parser( + "reconnect", parents=[hw_json], + help="Retry registering unregistered fleet nodes now (no leapd restart needed)", + ) + hw_fleet_reconnect.add_argument("node", nargs="?", default="", help="Configured node_id; omit for all") # Onboarding subcommands hw_init = hw_sub.add_parser("init", parents=[hw_json], help="Generate an annotated hc.v1 device template") hw_init.add_argument("device_id", help="Device id for the new template") diff --git a/src/leapflow/cli/commands/daemon.py b/src/leapflow/cli/commands/daemon.py index 7ae6ab4d..c750fd9d 100644 --- a/src/leapflow/cli/commands/daemon.py +++ b/src/leapflow/cli/commands/daemon.py @@ -196,7 +196,14 @@ async def _run() -> int: return asyncio.run(_run()) -def _stop(runtime_dir: Path, *, force: bool = False, timeout_s: float = 10.0) -> int: +def _stop( + runtime_dir: Path, + *, + force: bool = False, + timeout_s: float = 10.0, + action: str = "stop", + guarded: bool = False, +) -> int: from leapflow.daemon.lifecycle import DaemonInfo, cleanup_stale, stop_daemon info = DaemonInfo.discover(runtime_dir) @@ -208,9 +215,14 @@ def _stop(runtime_dir: Path, *, force: bool = False, timeout_s: float = 10.0) -> print("leapd is not running.") return 0 + if not guarded: + refusal = _stop_guard(info, action=action, force=force) + if refusal is not None: + return refusal + graceful_requested = False if info.is_healthy and info.sock_path is not None: - graceful_requested = _request_shutdown(info.sock_path) + graceful_requested = _request_shutdown(info.sock_path, action=action) print(f"Stopping leapd (pid={info.pid})...") result = stop_daemon( runtime_dir, @@ -230,12 +242,90 @@ def _stop(runtime_dir: Path, *, force: bool = False, timeout_s: float = 10.0) -> return 1 -def _request_shutdown(sock_path: Path) -> bool: +_SELF_TERMINATION_EXIT = 3 +HOST_DAEMON_PID_ENV = "LEAPFLOW_HOST_DAEMON_PID" + + +def _stop_guard(info: object, *, action: str, force: bool) -> int | None: + """Refuse a stop/restart that would kill a turn; return the exit code or None. + + Two layers. A process spawned by the very leapd it targets (an agent's + ``shell_run``, terminal session, MCP server -- all inherit + ``LEAPFLOW_HOST_DAEMON_PID``) is refused unconditionally, ``--force`` + included: the turn issuing the command runs inside that daemon and would + be torn down mid-flight, which is exactly how a detached ``nohup ... leap + daemon restart`` left the TUI with an unexplained dropped connection. + Otherwise, a daemon with active turns needs confirmation on a TTY and + ``--force`` elsewhere, since stopping it interrupts those turns. + """ + pid = getattr(info, "pid", None) + marker = os.environ.get(HOST_DAEMON_PID_ENV, "").strip() + if pid is not None and marker == str(pid): + sys.stderr.write( + f"Refusing 'leap daemon {action}': this command was launched from inside " + f"leapd (pid={pid}) itself, so it would terminate the agent turn that " + "issued it (--force does not override this).\n" + "- An unreachable fleet node is retried automatically; use " + "hw_fleet_reconnect (or 'leap hw fleet reconnect') to retry now.\n" + "- A restart-required setting: finish the turn and ask the user to run " + f"'leap daemon {action}' from their own terminal.\n" + ) + return _SELF_TERMINATION_EXIT + if force: + return None + active = _active_turns(info) + if not active: + return None + ids = ", ".join(active) + if sys.stdin is not None and sys.stdin.isatty(): + try: + answer = input( + f"leapd has {len(active)} active turn(s) ({ids}); '{action}' will " + "interrupt them and they may leave partial side effects. Continue? [y/N] " + ) + except EOFError: + answer = "" + if answer.strip().lower() in {"y", "yes"}: + return None + print(f"'leap daemon {action}' cancelled.") + return 1 + sys.stderr.write( + f"Refusing 'leap daemon {action}': leapd has {len(active)} active turn(s) " + f"({ids}) that would be interrupted. Wait for them to finish, cancel them " + "with /cancel, or pass --force to interrupt them.\n" + ) + return 1 + + +def _active_turns(info: object) -> list[str]: + """Return active turn request ids, or [] when leapd cannot be asked.""" + sock_path = getattr(info, "sock_path", None) + if not getattr(info, "is_healthy", False) or sock_path is None: + return [] + try: + status = asyncio.run(_runtime_status(sock_path)) + except Exception: # noqa: BLE001 - an unresponsive daemon must stay stoppable + return [] + admission = status.get("turn_admission") if isinstance(status, dict) else None + if not isinstance(admission, dict): + return [] + ids = [str(item) for item in admission.get("active_request_ids") or []] + count = int(admission.get("active", 0) or 0) + if count and not ids: + ids = [f"{count} unnamed"] + return ids + + +def _request_shutdown(sock_path: Path, *, action: str = "stop") -> bool: from leapflow.daemon.client import DaemonClient, DaemonUnavailableError + requested_by = {"pid": os.getpid(), "command": f"leap daemon {action}"} + async def _run() -> bool: try: - await DaemonClient(sock_path, timeout_s=2.0).shutdown() + await DaemonClient(sock_path, timeout_s=2.0).shutdown( + reason=f"leap daemon {action}", requested_by=requested_by + ) return True except DaemonUnavailableError: return False @@ -244,20 +334,31 @@ async def _run() -> bool: def _restart(settings: object, mock_host: bool, *, force: bool = False) -> int: + from leapflow.daemon.lifecycle import DaemonInfo + runtime_dir = settings.runtime_dir + info = DaemonInfo.discover(runtime_dir) + if info.is_running: + refusal = _stop_guard(info, action="restart", force=force) + if refusal is not None: + return refusal print("Restarting leapd...") - stop_code = _stop(runtime_dir, force=force, timeout_s=10.0) + stop_code = _stop(runtime_dir, force=force, timeout_s=10.0, action="restart", guarded=True) if stop_code != 0: sys.stderr.write("Restart aborted because old leapd did not stop cleanly.\n") return stop_code return _start(settings, mock_host) - async def _serve(settings: object, mock_host: bool) -> int: from leapflow.daemon.server import serve_daemon from leapflow.logging_setup import init_daemon_logging + # Every child leapd spawns (shell_run, terminal sessions, MCP servers, and + # anything they detach with nohup/setsid) inherits this marker, so a + # ``leap daemon stop|restart`` issued from inside a turn can recognise that + # it would kill its own host and refuse. + os.environ[HOST_DAEMON_PID_ENV] = str(os.getpid()) # The daemon writes stdout/stderr to leapd.log; without an explicit logging # setup Python's lastResort handler only emits WARNING+, hiding the INFO # field evidence (deferred init progress, turn usage). daemon.log_level is diff --git a/src/leapflow/cli/commands/hardware.py b/src/leapflow/cli/commands/hardware.py index cfc94446..6f9df85e 100644 --- a/src/leapflow/cli/commands/hardware.py +++ b/src/leapflow/cli/commands/hardware.py @@ -142,6 +142,8 @@ async def _dispatch(args: argparse.Namespace, action: str) -> int: return await _run_verification(action, args, json_mode) if action == "permit-key": return _run_permit_key(args, json_mode) + if action == "fleet": + return await _run_fleet(args, json_mode) if action in _ONBOARDING_ACTIONS: return await _run_onboarding(action, args, json_mode) _fail(f"Unknown hw action: {action}") @@ -271,6 +273,18 @@ async def _register_fleet_read_plane(registry: Any) -> None: async def _run_local(action: str, args: argparse.Namespace, json_mode: bool) -> int: settings = load_config() + if action == "list": + from leapflow.daemon.lifecycle import DaemonInfo + + info = DaemonInfo.discover(settings.runtime_dir) + if info.is_running: + return _emit_result(action, { + "ok": False, "code": "hardware_store_owned_by_daemon", + "failure_code": "hardware_store_owned_by_daemon", + "failure_class": "resource_busy", "side_effect_state": "none", + "error": f"leapd (pid={info.pid}) owns the hardware database; local reconciliation is unavailable.", + "next_step": "Use hw_list in the TUI or `leap daemon status`; do not restart leapd inside a turn.", + }, json_mode) registry = _build_local_registry(settings) if registry is None: return _emit_result( @@ -374,6 +388,41 @@ async def _run_sampling_control( return _emit_result(action, result, json_mode) +async def _run_fleet(args: argparse.Namespace, json_mode: bool) -> int: + """``leap hw fleet reconnect [node]`` -- retry node registration inside leapd. + + Fleet supervision lives only in leapd, so this always routes through its RPC; + retrying in this process would register the node into a throwaway registry. + """ + fleet_action = getattr(args, "fleet_action", None) + if fleet_action != "reconnect": + _fail("Usage: leap hw fleet reconnect [node]") + return 1 + node = str(getattr(args, "node", "") or "").strip() + action = "fleet reconnect" + info = _discover_daemon(load_config()) + if info is None: + return _emit_result( + action, + { + "ok": False, + "code": "daemon_required", + "error": ( + "Fleet nodes are supervised by leapd, which is not running. Start it with " + "`leap daemon start`; configured nodes register on startup and are " + "retried automatically while unreachable." + ), + }, + json_mode, + ) + client = DaemonClient(info.sock_path) + try: + result = await client.hardware_fleet_reconnect(node) + except DaemonUnavailableError as exc: + result = {"ok": False, "code": "daemon_error", "error": f"leapd request failed: {exc}"} + return _emit_result(action, result, json_mode) + + # ── Output ───────────────────────────────────────────────────────────────── @@ -417,6 +466,8 @@ def _render(action: str, result: dict[str, Any]) -> None: print(f"{_CYAN}LEAP Hardware \u2014 {action}{_RESET}") if not result.get("ok"): _fail(str(result.get("error") or result.get("code") or "command failed")) + if result.get("next_step"): + _info(str(result["next_step"])) return renderer = { "list": _render_list, @@ -427,11 +478,34 @@ def _render(action: str, result: dict[str, Any]) -> None: "pause": _render_sampling, "resume": _render_sampling, "replay": _render_replay, + "fleet reconnect": _render_fleet_reconnect, }.get(action) if renderer is not None: renderer(result) +def _render_fleet_reconnect(result: dict[str, Any]) -> None: + nodes = result.get("nodes") or {} + if not nodes: + _info("No unregistered fleet nodes; every configured node is registered.") + return + for node_id, outcome in nodes.items(): + status = str(outcome.get("status") or "") + if status == "online": + devices = ", ".join(outcome.get("device_ids") or []) or "(none)" + _ok(f"{node_id} online — devices: {devices}") + elif outcome.get("retry_in_flight"): + _info(f"{node_id}: a retry is already in progress") + elif status == "unregistered": + _warn( + f"{node_id} still unreachable: {outcome.get('last_error')} " + f"(attempt {outcome.get('attempts')}, next automatic retry in " + f"{outcome.get('next_retry_in_s')}s)" + ) + else: + _info(f"{node_id}: {status or outcome}") + + def _render_list(result: dict[str, Any]) -> None: devices = result.get("devices") or [] if not devices: diff --git a/src/leapflow/cli/commands/interactive.py b/src/leapflow/cli/commands/interactive.py index 0fe79e37..eec76a5a 100644 --- a/src/leapflow/cli/commands/interactive.py +++ b/src/leapflow/cli/commands/interactive.py @@ -1234,6 +1234,8 @@ async def _print_daemon_status() -> None: runtime_source = daemon_status.get("runtime_source") if runtime_source: console.system(f"Runtime source: {runtime_source}") + if daemon_status.get("runtime_installation_error"): + console.warning(str(daemon_status["runtime_installation_error"])) runtime_executable = daemon_status.get("runtime_executable") if runtime_executable: console.system(f"Python: {runtime_executable}") @@ -1274,7 +1276,7 @@ async def _stream_response( record_user: bool = True, ) -> None: nonlocal turn_count, active_turn_request_id - from leapflow.daemon.client import DaemonUnavailableError + from leapflow.daemon.client import DaemonRpcError, DaemonShutdownError, DaemonUnavailableError if record_user: exit_stats.record_user_message() @@ -1332,12 +1334,28 @@ async def _stream_response( if not metadata.get("heartbeat"): console.system(event.content) turn_completed = True + except DaemonRpcError: + console.warning( + "leapd returned a request error, not a connection loss. " + "The command was not replayed; inspect the error before trying again." + ) + raise except DaemonUnavailableError as exc: + if isinstance(exc, DaemonShutdownError): + # A deliberate stop is not a transport fault: replaying the turn would + # restart a daemon the user just stopped and could repeat side effects. + console.warning( + f"{exc.describe()}; this turn was interrupted and may have left " + "partial side effects. Check what it changed, then re-issue the " + "instruction (the TUI reconnects to leapd and keeps this session)." + ) + raise if allow_retry and not saw_real_event: retry_error = exc else: + progress = "after receiving turn output" if saw_real_event else "again before output" console.warning( - "Lost connection to leapd after the turn started; " + f"Lost connection to leapd {progress}; " "the command was not replayed to avoid duplicate side effects." ) raise @@ -1639,7 +1657,7 @@ def _handle_task_control(text: str) -> bool: async def _notification_listener() -> None: """Subscribe to daemon notifications and render them in TUI.""" nonlocal daemon_session_mode - from leapflow.daemon.client import DaemonUnavailableError + from leapflow.daemon.client import DaemonShutdownError, DaemonUnavailableError async def _refresh_watch_count() -> None: try: @@ -1691,10 +1709,10 @@ async def _refresh_watch_count() -> None: console.system("Distillation complete — no skills produced (insufficient signal)") app.invalidate() elif event_type == "monitor.finding": - console.monitor_card(payload) - if str(payload.get("severity")) == "alert": + # Defense in depth for notifications from older daemons. + if payload.get("severity") == "critical" and console.monitor_notice(payload): status.update_monitor_counts(alerts=status.alert_count + 1) - app.invalidate() + app.invalidate() elif event_type == "watch.state": await _refresh_watch_count() elif event_type == "signal.stream": @@ -1702,6 +1720,15 @@ async def _refresh_watch_count() -> None: status.increment_signal_stream() elif event_type == "monitor.error": logger.debug("monitor error notification: %s", payload) + except DaemonShutdownError as exc: + # A running turn reports this itself; an idle TUI would otherwise + # only find out on its next command. + if not app.agent_running: + console.warning( + f"{exc.describe()}. The next command reconnects to leapd " + "and keeps this session." + ) + await asyncio.sleep(3.0) except (DaemonUnavailableError, OSError, asyncio.IncompleteReadError): await asyncio.sleep(3.0) except asyncio.CancelledError: diff --git a/src/leapflow/cli/context.py b/src/leapflow/cli/context.py index 09cf6cb0..31157a10 100644 --- a/src/leapflow/cli/context.py +++ b/src/leapflow/cli/context.py @@ -1169,6 +1169,7 @@ def reload_runtime_config_if_changed(self, *, force: bool = False) -> bool: vlm=self.vlm, classifier=self.intent_classifier, ) + self._notify_runtime_config_reloaded() logger.info("Runtime configuration reloaded") return True @@ -1192,9 +1193,16 @@ def reload_runtime_config_if_changed(self, *, force: bool = False) -> bool: classifier=classifier, ) self._sync_engine_runtime_budget(updated) + self._notify_runtime_config_reloaded() logger.info("Runtime configuration reloaded") return True + def _notify_runtime_config_reloaded(self) -> None: + """Notify the daemon owner synchronously, including in-turn config writes.""" + callback = getattr(self, "_runtime_config_reload_callback", None) + if callback is not None: + callback(self) + def _host_backend_snapshot(self) -> dict[str, Any]: snapshot = getattr(self.rpc, "status_snapshot", None) if callable(snapshot): diff --git a/src/leapflow/cli/tui_app/console.py b/src/leapflow/cli/tui_app/console.py index 767fc3a9..c371f090 100644 --- a/src/leapflow/cli/tui_app/console.py +++ b/src/leapflow/cli/tui_app/console.py @@ -218,10 +218,27 @@ def command_footer(self, command: TuiCommand) -> None: label.append(f" {_format_elapsed(command.elapsed_s)}", style="leap.dim") self._console.print(label) + def monitor_notice(self, finding: Mapping[str, Any]) -> bool: + """Compact critical-only unsolicited notice; details remain in the board.""" + if finding.get("severity") != "critical": + return False + title = str(finding.get("title") or "Critical condition requires inspection")[:256] + notice = Text("Critical · ", style="leap.error") + notice.append(title, style="bold") + items = finding.get("items", []) + if isinstance(items, list): + scopes = [str(item.get("title", ""))[:160] for item in items[:3] if isinstance(item, Mapping)] + if scopes: + notice.append(" — " + "; ".join(scopes), style="leap.dim") + notice.append(" · Inspect /board list before continuing affected work.", style="leap.dim") + self._console.print(notice) + return True + def monitor_card(self, finding: Mapping[str, Any]) -> None: - """Render a compact, non-blocking card for a pushed monitor finding.""" + """Render a detailed card for explicit inspection.""" severity = str(finding.get("severity", "info")) border = { + "critical": "leap.error", "alert": "leap.error", "notable": "leap.warning", "info": "leap.border", diff --git a/src/leapflow/config.py b/src/leapflow/config.py index d164f508..9a75d248 100644 --- a/src/leapflow/config.py +++ b/src/leapflow/config.py @@ -302,6 +302,11 @@ class Settings: # endpoint, and token_env (preferred) or token. Empty retains the legacy # LEAPFLOW_FLEET_NODES environment fallback. hardware_fleet_nodes: str = "" + # A configured node unreachable at startup (or later dropped from the + # registry) is retried with exponential backoff between these bounds, so a + # rebooting edge board rejoins without a leapd restart. + hardware_fleet_retry_initial_s: float = 5.0 + hardware_fleet_retry_max_s: float = 60.0 # JSON bounded rank-only learning policy; never grants motion or trust. hardware_view_priors: str = "{}" # JSON array of event-to-action rules evaluated by hardware stream sources. @@ -920,6 +925,11 @@ class Settings: } ) + # Rolling plugin reliability policy; applies without restarting leapd. + plugin_health_error_rate_threshold: float = 0.25 + plugin_health_error_window_s: float = 3600.0 + plugin_health_min_calls: int = 5 + # ── Scheduler ── scheduler_enabled: bool = True scheduler_tick_seconds: int = 60 @@ -946,6 +956,7 @@ class Settings: dashboard_token_ref: str = "" # secret ref for the local dashboard access token # ── Session analysis dashboard (domain=session watch) ── + monitor_delivery: str = "{}" monitor_session_batch_turns: int = 6 monitor_session_batch_tokens: int = 4000 monitor_session_use_model_salience: bool = False @@ -1566,6 +1577,12 @@ def _build_settings_from_env( # (which persists into runtime.yaml) actually reaches Settings; the legacy # LEAPFLOW_FLEET_NODES fallback is applied later, in fleet_config. hardware_fleet_nodes = os.getenv("LEAPFLOW_HARDWARE_FLEET_NODES", "").strip() + hardware_fleet_retry_initial_s = float( + os.getenv("LEAPFLOW_HARDWARE_FLEET_RETRY_INITIAL_S", "5") or 5.0 + ) + hardware_fleet_retry_max_s = float( + os.getenv("LEAPFLOW_HARDWARE_FLEET_RETRY_MAX_S", "60") or 60.0 + ) hardware_view_priors = os.getenv("LEAPFLOW_HARDWARE_VIEW_PRIORS", "{}").strip() from leapflow.hardware.view_priors import ViewPriorSettings ViewPriorSettings.from_json(hardware_view_priors) @@ -1838,6 +1855,10 @@ def _tuple_env(key: str, default: tuple) -> tuple: except Exception: logger.warning("Invalid LEAPFLOW_OBSERVER_ENABLED_SET: %s", _observer_enabled_raw) + plugin_health_error_rate_threshold = float(os.getenv("LEAPFLOW_PLUGIN_HEALTH_ERROR_RATE_THRESHOLD", "0.25")) + plugin_health_error_window_s = float(os.getenv("LEAPFLOW_PLUGIN_HEALTH_ERROR_WINDOW_S", "3600")) + plugin_health_min_calls = int(os.getenv("LEAPFLOW_PLUGIN_HEALTH_MIN_CALLS", "5")) + # Scheduler scheduler_enabled = _bool("LEAPFLOW_SCHEDULER_ENABLED", "true") scheduler_tick_seconds = int(os.getenv("LEAPFLOW_SCHEDULER_TICK_SECONDS", "60")) @@ -1869,6 +1890,11 @@ def _tuple_env(key: str, default: tuple) -> tuple: except Exception: logger.warning("Invalid LEAPFLOW_USAGE_PRICING: %s", _raw_pricing) + # Domain-neutral presentation policy (reporting only, never safety authority). + monitor_delivery = os.getenv("LEAPFLOW_MONITOR_DELIVERY", "{}").strip() + from leapflow.monitor.delivery import DeliveryConfig + DeliveryConfig.parse(monitor_delivery) + # Session analysis dashboard monitor_session_batch_turns = int(os.getenv("LEAPFLOW_MONITOR_SESSION_BATCH_TURNS", "6")) monitor_session_batch_tokens = int(os.getenv("LEAPFLOW_MONITOR_SESSION_BATCH_TOKENS", "4000")) @@ -2141,6 +2167,8 @@ def _tuple_env(key: str, default: tuple) -> tuple: hardware_preview_quality=hardware_preview_quality, hardware_preview_idle_timeout_s=hardware_preview_idle_timeout_s, hardware_fleet_nodes=hardware_fleet_nodes, + hardware_fleet_retry_initial_s=hardware_fleet_retry_initial_s, + hardware_fleet_retry_max_s=hardware_fleet_retry_max_s, hardware_view_priors=hardware_view_priors, hardware_alert_policies=hardware_alert_policies, hardware_advisory_limits=hardware_advisory_limits, @@ -2274,6 +2302,9 @@ def _tuple_env(key: str, default: tuple) -> tuple: # Observer / Daemon observer_auto_start=observer_auto_start, observer_enabled_set=observer_enabled_set, + plugin_health_error_rate_threshold=plugin_health_error_rate_threshold, + plugin_health_error_window_s=plugin_health_error_window_s, + plugin_health_min_calls=plugin_health_min_calls, # Scheduler scheduler_enabled=scheduler_enabled, scheduler_tick_seconds=scheduler_tick_seconds, @@ -2292,6 +2323,7 @@ def _tuple_env(key: str, default: tuple) -> tuple: dashboard_token_ref=dashboard_token_ref, # Usage Pricing usage_pricing=usage_pricing, + monitor_delivery=monitor_delivery, monitor_session_batch_turns=monitor_session_batch_turns, monitor_session_batch_tokens=monitor_session_batch_tokens, monitor_session_use_model_salience=monitor_session_use_model_salience, diff --git a/src/leapflow/config_service.py b/src/leapflow/config_service.py index caaee56f..c724e1e9 100644 --- a/src/leapflow/config_service.py +++ b/src/leapflow/config_service.py @@ -3,6 +3,7 @@ from __future__ import annotations +import math import os import tempfile from dataclasses import dataclass, fields, replace @@ -158,9 +159,26 @@ class ConfigSnapshot: "survey signals; exact node/workcell configuration scope is mandatory. " "Never changes safety or approval. Requires a daemon restart." ), + "plugin_health.error_rate_threshold": "Failure fraction (0..1) above which recent plugin usage emits a health finding. Default 0.25; hot-reloaded.", + "plugin_health.error_window_s": "Rolling reliability window in positive seconds. Old usage outside the window is ignored. Default 3600; hot-reloaded.", + "plugin_health.min_calls": "Minimum recent samples before judging plugin reliability. Positive integer, default 5; hot-reloaded.", "hardware.fleet_nodes": ( - "JSON array of named LHP edge nodes. Connections and heartbeat supervision are " - "assembled at startup, so a change needs a daemon restart." + "JSON array of named LHP edge nodes. A configured node that is unreachable " + "(board rebooting, network down) is retried automatically with backoff and " + "rejoins without a restart; call hw_fleet_reconnect (or `leap hw fleet " + "reconnect`) to retry immediately. Changing node configuration (adding/removing " + "entries, endpoints, or credentials) needs a daemon restart, never from inside " + "an agent turn." + ), + "hardware.fleet_retry_initial_s": ( + "Seconds before the first registration retry of an unreachable fleet node; " + "each failure doubles the delay up to hardware.fleet_retry_max_s. Read when " + "fleet supervision starts, so a change needs a daemon restart." + ), + "hardware.fleet_retry_max_s": ( + "Upper bound, in seconds, of the backoff between registration retries of an " + "unreachable fleet node. Read when fleet supervision starts, so a change needs " + "a daemon restart." ), "hardware.alert_policies": ( "JSON array of hardware event-to-action rules. Empty disables alert reflexes; " @@ -299,10 +317,17 @@ class ConfigSnapshot: "else deletes from that table. Read when sampling starts, so a change needs a " "daemon restart." ), + "monitor.delivery": ( + "JSON presentation policy: minimum_severity (critical), global_interval_s (5), " + "repeat_interval_s (60), max_pending (128), max_items (5), max_bytes (8192), " + "updated_interval_s (1), max_state (4096). Persists evidence before gating; " + "valid hot reload preserves cooldowns and counters. Recovery is silent." + ), "hardware.observation": ( "JSON report-only policy with defaults and exact overrides (device_id/channel_id). " "soft_min/soft_max and margin_floor/margin_max are in channel units; confirm_s/recover_s " - "require continuous valid evidence. Adaptive defaults off and requires margin_max. " + "require continuous valid evidence. rate_confirm_s (0.5) and rate_recover_s (1.0) " + "debounce reporting only, never raw safety action. Adaptive defaults off and requires margin_max. " "Hard envelope and task accuracy are unchanged. Valid updates hot-reload; invalid " "updates retain the last admitted policy. Channel bounds are checked at hardware admission." ), @@ -795,6 +820,8 @@ def _spec_from_setting(name: str, value_type: Any) -> ConfigFieldSpec | None: if len(parts) == 1: return None section, field_name = parts + if name.startswith("plugin_health_"): + section, field_name = "plugin_health", name.removeprefix("plugin_health_") if section == "llm" and field_name == "api_key": return None if name in _SECRET_SETTINGS: @@ -894,6 +921,21 @@ def set( if spec.secret: return self._set_secret_backed_field(spec, str(value), scope=scope) coerced = _coerce_value(value, spec.value_type) + if normalized in { + "hardware.fleet_retry_initial_s", "hardware.fleet_retry_max_s", + "plugin_health.error_window_s", "plugin_health.min_calls", + "plugin_health.error_rate_threshold", + }: + if isinstance(value, bool) or not math.isfinite(coerced): + raise ValueError(f"{normalized} must be finite numeric data") + if normalized == "plugin_health.error_rate_threshold": + if not 0 <= coerced <= 1: + raise ValueError(f"{normalized} must be within 0..1") + elif coerced <= 0: + raise ValueError(f"{normalized} must be positive") + if normalized == "monitor.delivery": + from leapflow.monitor.delivery import DeliveryConfig + DeliveryConfig.parse(coerced) if normalized == "hardware.observation": from leapflow.hardware.observation import ObservationSettings policy = ObservationSettings.parse(coerced) @@ -1083,7 +1125,9 @@ def _restart_warnings(spec: ConfigFieldSpec) -> tuple[str, ...]: return () return ( f"{spec.key} requires `leap daemon restart` to take effect; " - "the running leapd keeps its previous value until restarted.", + "the running leapd keeps its previous value until restarted. " + "An agent must not restart leapd from inside its own turn (that terminates " + "the turn); the user runs the restart after the turn ends.", ) diff --git a/src/leapflow/daemon/_service_helpers.py b/src/leapflow/daemon/_service_helpers.py index 19645544..a336bcd7 100644 --- a/src/leapflow/daemon/_service_helpers.py +++ b/src/leapflow/daemon/_service_helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +from pathlib import Path from typing import Any, TYPE_CHECKING from leapflow.engine import StreamEvent @@ -171,6 +172,26 @@ def runtime_source() -> str: return str(getattr(leapflow, "__file__", "")) +def runtime_installation_error() -> str: + """Detect a package removed beneath this process without importing new code. + + A wheel-to-editable reinstall can delete the running package's directory. + Already loaded modules continue working, but lazy imports use the old path. + Unknown origins (for example an embedded runtime) are not called broken. + """ + source = runtime_source() + if not source: + return "" + if Path(source).is_file(): + return "" + return ( + "LeapFlow's running package files are no longer available; its installation " + "changed while leapd was running. This request was not started. Finish any " + "other active work, then run `leap daemon restart` from your own terminal " + "to load the current installation. Do not restart from inside an agent turn." + ) + + def runtime_version() -> str: try: from leapflow.version import __version__ diff --git a/src/leapflow/daemon/client.py b/src/leapflow/daemon/client.py index 3b2ea473..fc5371f5 100644 --- a/src/leapflow/daemon/client.py +++ b/src/leapflow/daemon/client.py @@ -19,8 +19,15 @@ stop_daemon, wait_ready, ) -from leapflow.daemon.protocol import RpcRequest +from leapflow.daemon.protocol import ErrorCode, RpcError, RpcRequest from leapflow.engine import StreamEvent +from leapflow.engine.recovery.failure_envelope import ( + FailureContext, + FailureEnvelope, + FailureSource, + Recoverability, + SideEffectState, +) logger = logging.getLogger(__name__) @@ -31,6 +38,91 @@ class DaemonUnavailableError(RuntimeError): """Raised when a usable leapd daemon cannot be reached.""" +class DaemonRpcError(RuntimeError): + """A reachable daemon rejected or failed an RPC; reconnecting cannot fix it. + + Keep the wire code/data and a non-replayable failure envelope. A backend + response does not prove that no side effects occurred, even before output. + """ + + def __init__(self, error: object, *, method: str = "", request_id: str = "") -> None: + payload = error if isinstance(error, dict) else {} + code = payload.get("code") + self.error = RpcError( + code=code if isinstance(code, int) else ErrorCode.INTERNAL_ERROR, + message=str(payload.get("message") or "Daemon request failed"), + data=payload.get("data") if payload else error, + ) + self.code = self.error.code + self.data = self.error.data + message = _format_rpc_error(error) + self.failure = FailureEnvelope.create( + source=FailureSource.SYSTEM, + category="internal_defect" if self.code == ErrorCode.INTERNAL_ERROR else "daemon_rpc", + failure_class="daemon_rpc_error", + failure_code=str(self.code), + message=message, + recoverability=Recoverability.NON_RECOVERABLE, + side_effect_state=SideEffectState.UNKNOWN, + context=FailureContext(tool_name=method, execution_id=request_id), + ) + super().__init__(message) + + +class DaemonShutdownError(DaemonUnavailableError): + """leapd announced a deliberate shutdown before closing this connection. + + Carries who stopped it and why, so a client can say "leapd was stopped by + `leap daemon restart` (pid X)" instead of an unexplained dropped connection. + """ + + def __init__( + self, + reason: str, + *, + requested_by_pid: int | None = None, + requested_by_command: str = "", + active_turns: int = 0, + ) -> None: + self.reason = reason or "unknown" + self.requested_by_pid = requested_by_pid + self.requested_by_command = requested_by_command + self.active_turns = active_turns + super().__init__(self.describe()) + + def describe(self) -> str: + """Return a one-line account of the shutdown.""" + if not self.requested_by_command: + return f"leapd shut down ({self.reason})" + pid = f" (pid {self.requested_by_pid})" if self.requested_by_pid else "" + return f"leapd was stopped by `{self.requested_by_command}`{pid}" + + @classmethod + def from_metadata(cls, metadata: dict[str, Any]) -> "DaemonShutdownError": + pid = metadata.get("requested_by_pid") + try: + pid = int(pid) if pid is not None else None + except (TypeError, ValueError): + pid = None + return cls( + str(metadata.get("reason") or ""), + requested_by_pid=pid, + requested_by_command=str(metadata.get("requested_by_command") or ""), + active_turns=int(metadata.get("active_turns") or 0), + ) + + +def _raise_if_shutdown(payload: dict[str, Any]) -> None: + """Raise :class:`DaemonShutdownError` for a ``daemon_shutdown`` notification.""" + if payload.get("method") != "stream.chunk": + return + params = payload.get("params") or {} + if params.get("event_type") != "daemon_shutdown": + return + metadata = params.get("metadata") + raise DaemonShutdownError.from_metadata(metadata if isinstance(metadata, dict) else {}) + + class DaemonClient: """Small JSON-RPC client that opens one Unix socket per request.""" @@ -62,6 +154,7 @@ async def request( await _send(writer, request.to_json()) while True: payload = await self._read_payload(reader) + _raise_if_shutdown(payload) # Skip heartbeat notifications sent by the server for # long-running handlers — they carry no result. if payload.get("method") == "stream.chunk": @@ -77,7 +170,7 @@ async def request( if payload.get("id") != request.id: continue if "error" in payload: - raise DaemonUnavailableError(_format_rpc_error(payload["error"])) + raise DaemonRpcError(payload["error"], method=request.method, request_id=request.id) return payload.get("result") finally: await _close_writer(writer) @@ -118,6 +211,7 @@ async def engine_chat( await _send(writer, request.to_json()) while True: payload = await self._read_payload(reader) + _raise_if_shutdown(payload) method = payload.get("method") params = dict(payload.get("params") or {}) if method == "stream.chunk" and params.get("id") == request.id: @@ -127,7 +221,7 @@ async def engine_chat( continue if payload.get("id") == request.id: if "error" in payload: - raise DaemonUnavailableError(_format_rpc_error(payload["error"])) + raise DaemonRpcError(payload["error"], method=request.method, request_id=request.id) break finally: await _close_writer(writer) @@ -144,6 +238,7 @@ async def subscribe_notifications(self) -> AsyncIterator[dict[str, Any]]: await _send(writer, request.to_json()) while True: payload = await self._read_payload(reader) + _raise_if_shutdown(payload) method = payload.get("method") params = dict(payload.get("params") or {}) if method == "stream.chunk" and params.get("id") == request.id: @@ -154,6 +249,8 @@ async def subscribe_notifications(self) -> AsyncIterator[dict[str, Any]]: yield metadata continue if payload.get("id") == request.id: + if "error" in payload: + raise DaemonRpcError(payload["error"], method=request.method, request_id=request.id) break finally: await _close_writer(writer) @@ -237,6 +334,11 @@ async def hardware_resume(self, device: str) -> dict[str, Any]: result = await self.request("hardware.resume", {"device": device}) return dict(result or {}) + async def hardware_fleet_reconnect(self, node: str = "") -> dict[str, Any]: + """Retry registering unregistered fleet nodes inside leapd.""" + result = await self.request("hardware.fleet_reconnect", {"node": node}) + return dict(result or {}) + async def hardware_inventory(self) -> dict[str, Any]: """Return the admitted device fleet grouped by declared class.""" result = await self.request("hardware.inventory") @@ -318,6 +420,7 @@ async def hardware_preview_stream( await _send(writer, request.to_json()) while True: payload = await self._read_payload(reader) + _raise_if_shutdown(payload) params = dict(payload.get("params") or {}) if payload.get("method") == "stream.chunk" and params.get("id") == request.id: metadata = params.get("metadata") or {} @@ -331,9 +434,10 @@ async def hardware_preview_stream( await result continue if event_type == "error": - raise DaemonUnavailableError( - str(metadata.get("error") or params.get("content") or "preview stream failed") - ) + raise DaemonRpcError({ + "code": ErrorCode.INTERNAL_ERROR, + "message": str(metadata.get("error") or params.get("content") or "preview stream failed"), + }, method=request.method, request_id=request.id) if params.get("done"): continue if event_type == "frame": @@ -341,7 +445,7 @@ async def hardware_preview_stream( continue if payload.get("id") == request.id: if "error" in payload: - raise DaemonUnavailableError(_format_rpc_error(payload["error"])) + raise DaemonRpcError(payload["error"], method=request.method, request_id=request.id) return finally: await _close_writer(writer) @@ -591,9 +695,18 @@ async def subagent_state(self) -> dict[str, Any]: """Fetch subagent delegation state from daemon.""" return dict(await self.request("subagent.state") or {}) - async def shutdown(self) -> None: - """Request graceful daemon shutdown.""" - await self.request("daemon.shutdown") + async def shutdown(self, *, reason: str = "", requested_by: dict[str, Any] | None = None) -> None: + """Request graceful daemon shutdown, naming who asked and why. + + leapd forwards both to every client with an open stream before closing + it, so their users see why the connection ended. + """ + params: dict[str, Any] = {} + if reason: + params["reason"] = reason + if requested_by: + params["requested_by"] = dict(requested_by) + await self.request("daemon.shutdown", params) async def _open(self) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: try: diff --git a/src/leapflow/daemon/monitor_coordinator.py b/src/leapflow/daemon/monitor_coordinator.py index 18a19adf..edcc4c3f 100644 --- a/src/leapflow/daemon/monitor_coordinator.py +++ b/src/leapflow/daemon/monitor_coordinator.py @@ -14,6 +14,7 @@ from leapflow.daemon._transport import RPC_STREAM_LIMIT from leapflow.daemon.notifications import Notification +from leapflow.monitor.delivery import BoundedFindingDelivery, DeliveryConfig from leapflow.monitor.signal_noise import SignalNoiseConfig, SignalNoiseGate logger = logging.getLogger(__name__) @@ -78,11 +79,17 @@ async def start(self, ctx: Any, notification_bus: Any, settings: Any) -> None: services=self._build_services_proxy(ctx, settings), tick_seconds=int(getattr(settings, "scheduler_tick_seconds", 120)), grace_seconds=float(getattr(settings, "scheduler_grace_seconds", 120.0)), + run_db=self._off_loop, + delivery_factory=lambda store, emit, validate: BoundedFindingDelivery( + store, emit, config=DeliveryConfig.parse(getattr(settings, "monitor_delivery", "{}")), + run_db=self._off_loop, validate=validate), ) self._monitors.producers.register(SessionAnalysisProducer()) self._monitors.producers.register(SignalObservationProducer()) self._monitors.producers.register(CapabilityAdaptationProducer()) - self._monitors.producers.register(PluginHealthProducer()) + self._plugin_health_producer = PluginHealthProducer() + self._plugin_health_producer.configure_from_settings(settings) + self._monitors.producers.register(self._plugin_health_producer) self._monitors.producers.register(EvolutionProducer()) self._register_hardware_producer(ctx, settings) self._install_evolution_sink(ctx, settings) @@ -275,9 +282,14 @@ def _build_services_proxy(self, ctx: Any, settings: Any) -> Any: def update_settings(self, settings: Any) -> None: """Hot-apply signal noise policy settings to the running gate.""" + if self._monitors is not None: + self._monitors.delivery.configure(getattr(settings, "monitor_delivery", "{}")) gate = self._signal_noise_gate if gate is not None: gate.update_config(SignalNoiseConfig.from_settings(settings)) + producer = getattr(self, "_plugin_health_producer", None) + if producer is not None: + producer.configure_from_settings(settings) def _make_monitor_signal_subscriber( self, @@ -374,6 +386,7 @@ def signal_noise_stats(self) -> dict[str, Any]: # ``hardware.enabled`` so the board has a watch to report against; with the # producer unregistered the cycle is a no-op. ("hardware-bench", "hardware", "2m", False), + ("hardware-live", "hardware", "event:hw.*", False), # Framework self-evolution, armed twice on purpose. The domain answers two # different questions with two different cadences: a *state* snapshot (what is # registered, what trust each plugin holds, which pipeline segments show @@ -418,8 +431,8 @@ async def _arm_default_watches(self) -> None: entry = existing.get(name) if entry is not None: view, is_active = entry - if is_active: - # Already armed and active — nothing to do. + if is_active or str(view.state) == "suspended": + # Preserve existing active watches and deliberate user pauses. continue # Stale (done/failed/suspended) — remove so we can re-arm. try: @@ -631,6 +644,7 @@ async def get_summary(self) -> dict[str, Any]: "active": len(active), "standalone_active": len(standalone), "client_coupled_active": len(coupled), + "delivery": monitors.delivery.stats, "active_samples": [ { "watch_id": str(watch.get("watch_id", "")), diff --git a/src/leapflow/daemon/notifications.py b/src/leapflow/daemon/notifications.py index 3a080565..e57f4887 100644 --- a/src/leapflow/daemon/notifications.py +++ b/src/leapflow/daemon/notifications.py @@ -37,28 +37,91 @@ def to_dict(self) -> Dict[str, Any]: } -class NotificationBus: - """Broadcast notifications to all subscribed TUI clients. +_REPLACEABLE = frozenset({"monitor.updated", "signal.stream", "view.resync", "teach.progress"}) + + +def _priority(notification: Optional[Notification]) -> int: + if notification is None or notification.event_type in {"daemon.shutdown", "approval_request"}: + return 2 + if notification.event_type == "monitor.finding" and notification.payload.get("severity") == "critical": + return 2 + return 0 if notification.event_type in _REPLACEABLE else 1 + + +def _replacement_key(notification: Notification) -> tuple: + payload = notification.payload + return (notification.event_type, payload.get("watch_id"), payload.get("session_id"), + payload.get("request_id")) + + +class _NotificationQueue(asyncio.Queue): + """Priority insertion via Queue's supported subclass hook.""" + + def _put(self, item: Optional[Notification]) -> None: + priority = _priority(item) + for index, queued in enumerate(self._queue): + if _priority(queued) < priority: + self._queue.insert(index, item) + return + self._queue.append(item) + + def offer(self, item: Notification) -> tuple[int, int, bool]: + """Replace/evict queued work without completing retained work for join().""" + coalesced = overflowed = 0 + if item.event_type in _REPLACEABLE: + key = _replacement_key(item) + for queued in list(self._queue): + if queued is not None and _replacement_key(queued) == key: + self._queue.remove(queued) + coalesced += 1 + if self.full(): + victim = next((queued for queued in reversed(self._queue) + if _priority(queued) < _priority(item)), None) + if victim is None: + return coalesced, 1, False + self._queue.remove(victim) + overflowed = 1 + self.put_nowait(item) + # Account for discarded work only after the new work is pending. + for _ in range(coalesced + overflowed): + self.task_done() + return coalesced, overflowed, True + + def terminate(self) -> int: + removed = len(self._queue) + self._queue.clear() + self.put_nowait(None) + for _ in range(removed): + self.task_done() + return removed + - Thread-safe via asyncio.Queue per subscriber. - """ +class NotificationBus: + """Event-loop-owned bounded fan-out; a slow client cannot stall another.""" def __init__(self, maxsize: int = 64) -> None: - self._subscribers: Dict[str, asyncio.Queue[Optional[Notification]]] = {} + self._subscribers: Dict[str, _NotificationQueue] = {} + if type(maxsize) is not int or maxsize <= 0: + raise ValueError("notification queue capacity must be a positive integer") self._maxsize = maxsize + self._stats = {"coalesced": 0, "overflowed": 0, "enqueued": 0} def subscribe(self, subscriber_id: str) -> asyncio.Queue[Optional[Notification]]: """Register a subscriber and return its queue. Send None to the queue to signal graceful shutdown. """ - q: asyncio.Queue[Optional[Notification]] = asyncio.Queue(maxsize=self._maxsize) + if subscriber_id in self._subscribers: + self._terminate(self._subscribers[subscriber_id]) + q = _NotificationQueue(maxsize=self._maxsize) self._subscribers[subscriber_id] = q logger.debug("notification_bus: subscriber added id=%s total=%d", subscriber_id, len(self._subscribers)) return q - def unsubscribe(self, subscriber_id: str) -> None: - """Remove a subscriber.""" + def unsubscribe(self, subscriber_id: str, queue: asyncio.Queue | None = None) -> None: + """Remove only the subscription owned by the closing stream.""" + if queue is not None and self._subscribers.get(subscriber_id) is not queue: + return self._subscribers.pop(subscriber_id, None) logger.debug("notification_bus: subscriber removed id=%s total=%d", subscriber_id, len(self._subscribers)) @@ -69,11 +132,11 @@ def emit(self, notification: Notification) -> None: subscriber (back-pressure). This prevents slow clients from blocking background tasks. """ - for sid, q in list(self._subscribers.items()): - try: - q.put_nowait(notification) - except asyncio.QueueFull: - logger.debug("notification_bus: dropped event for slow subscriber %s", sid) + for q in list(self._subscribers.values()): + coalesced, overflowed, enqueued = q.offer(notification) + self._stats["coalesced"] += coalesced + self._stats["overflowed"] += overflowed + self._stats["enqueued"] += int(enqueued) def emit_event(self, event_type: str, **payload: Any) -> None: """Convenience: emit with keyword args.""" @@ -83,11 +146,16 @@ def emit_event(self, event_type: str, **payload: Any) -> None: def subscriber_count(self) -> int: return len(self._subscribers) + @property + def stats(self) -> dict[str, int]: + """Fan-out enqueue/overflow counts, not user receipt acknowledgments.""" + return dict(self._stats) + + def _terminate(self, q: _NotificationQueue) -> None: + self._stats["overflowed"] += q.terminate() + async def shutdown(self) -> None: - """Signal all subscribers to disconnect.""" + """Guaranteed sentinel even when every subscriber queue is full.""" for q in self._subscribers.values(): - try: - q.put_nowait(None) - except asyncio.QueueFull: - pass + self._terminate(q) self._subscribers.clear() diff --git a/src/leapflow/daemon/protocol.py b/src/leapflow/daemon/protocol.py index aae2348b..4b5cec9c 100644 --- a/src/leapflow/daemon/protocol.py +++ b/src/leapflow/daemon/protocol.py @@ -115,6 +115,7 @@ class StreamChunk: "final", "tool_start", "tool_complete", + "daemon_shutdown", "thinking", "status", "error", @@ -348,6 +349,14 @@ async def hardware_rescan(self) -> Dict[str, Any]: """ ... + async def hardware_fleet_reconnect(self, node: str = "") -> Dict[str, Any]: + """Retry registering configured fleet nodes that are not registered. + + Daemon-global like ``hardware_rescan``; connects and reads declarations + only, never actuates, so it takes no turn-admission slot or approval. + """ + ... + async def hardware_frame( self, device: str = "", @@ -448,8 +457,10 @@ async def approval_cancel(self, pending_id: str, reason: str = "cancelled") -> D """Cancel a pending approval request.""" ... - async def shutdown(self) -> None: - """Graceful shutdown.""" + async def shutdown( + self, *, reason: str = "leapd stopping", requested_by: Dict[str, Any] | None = None + ) -> None: + """Announce the reason to clients before graceful shutdown.""" ... # ── Gateway ────────────────────────────────────────────────────── @@ -522,6 +533,7 @@ async def gateway_send( "hardware.inventory": "hardware_inventory", "hardware.device": "hardware_device", "hardware.rescan": "hardware_rescan", + "hardware.fleet_reconnect": "hardware_fleet_reconnect", "hardware.frame": "hardware_frame", "hardware.preview.stream": "hardware_preview_stream", "hardware.preview.release": "hardware_preview_release", diff --git a/src/leapflow/daemon/reentry_coordinator.py b/src/leapflow/daemon/reentry_coordinator.py index a24375f6..0c507eed 100644 --- a/src/leapflow/daemon/reentry_coordinator.py +++ b/src/leapflow/daemon/reentry_coordinator.py @@ -46,6 +46,9 @@ async def start( ``agent_reentry_enabled`` (default off) plus a global-budget backstop. Best-effort: never blocks startup. """ + if self.is_running(): + return + self._ctx = ctx try: store = getattr(ctx, "_reentry_store", None) manager = getattr(ctx, "_subagent_manager", None) @@ -145,6 +148,15 @@ async def _loop() -> None: async def stop(self) -> None: """Stop the reentry driver and wait for task completion.""" + ctx = getattr(self, "_ctx", None) + if ctx is not None: + ctx._reentry_event_observer = None + ctx._reentry_hardware_observer = None + registry = getattr(ctx, "_hardware_registry", None) + if registry is not None: + registry.set_deliberative_observer(None) + self._ctx = None + self._reentry_service = None if self._reentry_stop is not None: self._reentry_stop.set() if self._reentry_task is not None: diff --git a/src/leapflow/daemon/server.py b/src/leapflow/daemon/server.py index a474e8df..3f3240f0 100644 --- a/src/leapflow/daemon/server.py +++ b/src/leapflow/daemon/server.py @@ -95,6 +95,11 @@ def __init__( # Strong refs for stream finalizers that outlive a cancelled connection. self._stream_closers: set[asyncio.Task[Any]] = set() self._stopping = False + # RPC ids are connection-local; key by writer to preserve every client. + self._streams: dict[asyncio.StreamWriter, tuple[str, str]] = {} + self._shutdown_announced = False + if hasattr(service, "set_shutdown_notifier"): + service.set_shutdown_notifier(self.announce_shutdown) if hasattr(service, "set_client_count_provider"): service.set_client_count_provider(lambda: self._active_connections) if hasattr(service, "set_client_lease_provider"): @@ -141,6 +146,48 @@ async def serve_forever(self) -> None: finally: transport.cleanup(self._runtime_dir) + async def announce_shutdown( + self, reason: str, requested_by: dict[str, Any] | None = None + ) -> int: + """Tell every open stream why leapd is about to close it; return how many. + + Without this a client only sees EOF and reports "leapd closed the + connection unexpectedly", indistinguishable from a crash. Sent once, + before the service is torn down, so it reaches streams whose turn is + still running. Best effort per stream: a client that already hung up + or stopped reading must not delay the shutdown. + """ + if self._shutdown_announced: + return 0 + self._shutdown_announced = True + by = dict(requested_by or {}) + streams = list(self._streams.items()) + active_turns = sum(1 for _, (_, method) in streams if method == "engine.chat") + logger.info( + "daemon: shutting down reason=%s requested_by=%s open_streams=%d active_turns=%d", + reason, by or "-", len(streams), active_turns, + ) + sent = 0 + for writer, (request_id, method) in streams: + notification = StreamChunk( + request_id=request_id, + content=f"leapd is shutting down: {reason}", + event_type="daemon_shutdown", + metadata={ + "reason": reason, + "requested_by_pid": by.get("pid"), + "requested_by_command": str(by.get("command") or ""), + "active_turns": active_turns, + "interrupted_turn": method == "engine.chat", + }, + ).to_notification() + try: + await asyncio.wait_for(_write_json(writer, notification.to_json()), timeout=1.0) + sent += 1 + except (_ClientDisconnected, TimeoutError, OSError, RuntimeError): + logger.debug("daemon: shutdown notice not delivered request_id=%s", request_id) + return sent + async def stop(self) -> None: """Close client lifetimes before waiting for the listening server to drain.""" self._stopping = True @@ -359,6 +406,7 @@ async def _dispatch_stream( # copied Context makes ContextVar reset tokens invalid on the next generator step. ctx = contextvars.copy_context() completed = False + self._streams[writer] = (request.id, request.method) try: stream = method(**params) pending = asyncio.create_task(anext(stream), context=ctx) @@ -416,6 +464,7 @@ async def _dispatch_stream( except (ConnectionResetError, BrokenPipeError): pass finally: + self._streams.pop(writer, None) if approval_get is not None and not approval_get.done(): approval_get.cancel() if pending is not None and not pending.done(): @@ -487,8 +536,11 @@ async def serve_daemon(settings: Any, *, mock_host: bool = False) -> int: await service.start() loop = asyncio.get_running_loop() stop_event = asyncio.Event() + stop_reason = [""] - def _request_stop() -> None: + def _request_stop(reason: str = "") -> None: + if reason and not stop_reason[0]: + stop_reason[0] = reason stop_event.set() server = UnixRpcServer( @@ -500,7 +552,7 @@ def _request_stop() -> None: for sig in (signal.SIGINT, signal.SIGTERM): try: - loop.add_signal_handler(sig, _request_stop) + loop.add_signal_handler(sig, _request_stop, f"signal {sig.name}") except NotImplementedError: logger.debug("daemon: signal handlers unsupported on this event loop") @@ -522,6 +574,9 @@ def _request_stop() -> None: await idle_task except asyncio.CancelledError: pass + # A daemon.shutdown RPC has already announced itself (no-op here); a + # signal still owes open streams an explanation before they close. + await server.announce_shutdown(stop_reason[0] or "leapd stopping") # serve_forever itself awaits wait_closed when cancelled on Python # 3.12+, so accepted clients must be closed before cancelling that task. await server.stop() @@ -567,6 +622,7 @@ async def _watch_idle_shutdown( last_active = asyncio.get_running_loop().time() elif asyncio.get_running_loop().time() - last_active >= idle_timeout_s: logger.info("daemon: idle timeout reached; shutting down") + await server.announce_shutdown("idle timeout") stop_event.set() return await asyncio.sleep(interval) diff --git a/src/leapflow/daemon/service.py b/src/leapflow/daemon/service.py index 54cdc391..29a4848e 100644 --- a/src/leapflow/daemon/service.py +++ b/src/leapflow/daemon/service.py @@ -32,6 +32,7 @@ memory_entry_to_dict, normalize_stream_event, persisted_session_workspace, + runtime_installation_error, runtime_source, runtime_version, ) @@ -465,6 +466,10 @@ def __init__(self, settings: Any, *, mock_host: bool = False, auto_start_deferre from leapflow.daemon.notifications import NotificationBus self.notification_bus = NotificationBus() + def set_shutdown_notifier(self, notifier: Callable[..., Any]) -> None: + """Bind the transport-owned shutdown announcement before teardown.""" + self._shutdown_notifier = notifier + def set_client_count_provider(self, provider: Callable[[], int]) -> None: self._client_count = provider @@ -484,6 +489,8 @@ async def start(self) -> None: self._approval_coordinator.install_gate(ctx, self) install_learn_notifications(ctx, self.notification_bus) self._ctx = ctx + ctx._runtime_config_reload_callback = self._on_runtime_config_reloaded + self._reentry_reload_task: asyncio.Task[Any] | None = None # Wire the session-scoped status snapshot into the self-awareness surface # so self_describe/runtime_snapshot answer from live evidence rather than # source code. bind_runtime reaches only plugins declaring @@ -597,12 +604,22 @@ def start_deferred_init(self) -> None: if self._deferred_init_task is None or self._deferred_init_task.done(): self._deferred_init_task = asyncio.create_task(self._run_deferred_init(self._ctx)) - async def shutdown(self) -> None: + async def shutdown( + self, *, reason: str = "leapd stopping", requested_by: dict[str, Any] | None = None + ) -> None: + notifier = getattr(self, "_shutdown_notifier", None) + if notifier is not None: + await notifier(reason, requested_by) self._build_staleness.cancel_pending() if self._ctx is None: return ctx = self._ctx self._ctx = None + ctx._runtime_config_reload_callback = None + reload_task = getattr(self, "_reentry_reload_task", None) + if reload_task is not None and not reload_task.done(): + reload_task.cancel() + await asyncio.gather(reload_task, return_exceptions=True) environment_sources = self._environment_source_manager self._environment_source_manager = None if environment_sources is not None: @@ -687,6 +704,11 @@ def _session_registry(self, value: Any) -> None: async def engine_chat(self, message: str, **kwargs: Any) -> AsyncIterator[StreamChunk]: request_id = str(kwargs.get("request_id") or uuid.uuid4().hex[:12]) workspace_arg = str(kwargs.get("workspace_root") or "").strip() + installation_error = await asyncio.to_thread(runtime_installation_error) + if installation_error: + # Refuse before warming, admission, session construction, or engine I/O. + # A reinstall changes future imports, not this process's package path. + raise RuntimeError(installation_error) # Ensure deferred init completed. Emit a status chunk first so the # server dispatch loop receives an immediate first chunk (keepalive @@ -784,9 +806,6 @@ async def engine_chat(self, message: str, **kwargs: Any) -> AsyncIterator[Stream ctx = self.context try: if ctx.reload_runtime_config_if_changed(): - self._settings = ctx.settings - self._monitor_coordinator.update_settings(ctx.settings) - self._propagate_config_to_sessions(ctx) chunk = StreamChunk( request_id=request_id, content="Configuration reloaded in leapd.", @@ -1142,6 +1161,32 @@ async def evolution_projection_aggregate( def _ensure_session_registry(self, base_engine: Any) -> Any: return self._session_coordinator.ensure_registry(base_engine, self._settings) + def _on_runtime_config_reloaded(self, ctx: Any) -> None: + """Apply config to every live session before config_set returns.""" + self._settings = ctx.settings + self._propagate_config_to_sessions(ctx) + self._monitor_coordinator.update_settings(ctx.settings) + task = getattr(self, "_reentry_reload_task", None) + if task is None or task.done(): + self._reentry_reload_task = asyncio.create_task(self._sync_reentry_runtime(ctx)) + + async def _sync_reentry_runtime(self, ctx: Any) -> None: + """Converge lifecycle to the latest setting; rapid toggles cannot spawn twice.""" + try: + while self._ctx is ctx: + enabled = bool(getattr(self._settings, "agent_reentry_enabled", False)) + if enabled and not self._reentry_coordinator.is_running(): + await self._reentry_coordinator.start( + ctx, self._settings, self._turn_admission, self.notification_bus, + request_approval=self._request_approval, + ) + elif not enabled and self._reentry_coordinator.is_running(): + await self._reentry_coordinator.stop() + if enabled == bool(getattr(self._settings, "agent_reentry_enabled", False)): + return + except Exception: + logger.warning("Reentry runtime config reload failed", exc_info=True) + def _propagate_config_to_sessions(self, ctx: Any) -> None: """Propagate updated LLM/config to all active session engines.""" registry = self._session_coordinator.registry @@ -1151,6 +1196,7 @@ def _propagate_config_to_sessions(self, ctx: Any) -> None: llm = ctx.llm vlm = getattr(ctx, "vlm", None) classifier = getattr(ctx, "intent_classifier", None) + from leapflow.engine.session.session_factory import _settings_for_workspace from leapflow.memory.providers.working import resolve_working_memory_max_tokens for sid in registry.session_ids(): @@ -1162,8 +1208,9 @@ def _propagate_config_to_sessions(self, ctx: Any) -> None: # Base engine is already reconfigured by reload_runtime_config_if_changed. continue try: + session_settings = _settings_for_workspace(settings, session_ctx.workspace_root) engine.reconfigure_runtime( - settings=settings, + settings=session_settings, llm=llm, vlm=vlm, classifier=classifier, @@ -1418,6 +1465,24 @@ async def hardware_rescan(self) -> dict[str, Any]: } return {"ok": True, **report.to_dict()} + async def hardware_fleet_reconnect(self, node: str = "") -> dict[str, Any]: + """Retry registering configured fleet nodes that are not registered yet. + + The supported recovery for a node that was unreachable when leapd started: + unlike a daemon restart it interrupts no turn. Connects and reads + declarations only, so it takes no turn-admission slot and no approval. + """ + _, refusal = self._hardware_registry_or_error("a fleet reconnect") + if refusal is not None: + return refusal + from leapflow.plugins import get_registry as _get_tool_registry + + plugin = _get_tool_registry().get_plugin("hardware_context") + reconnect = getattr(plugin, "fleet_reconnect", None) + if not callable(reconnect): + return {"ok": False, "code": "fleet_unavailable", "error": "The hardware plugin is not loaded."} + return await reconnect(str(node or "").strip() or None) + async def hardware_frame( self, device: str = "", @@ -1756,7 +1821,7 @@ async def subscribe_notifications(self) -> AsyncIterator[StreamChunk]: metadata=notification.to_dict(), ) finally: - self.notification_bus.unsubscribe(subscriber_id) + self.notification_bus.unsubscribe(subscriber_id, queue) # ── Status ─────────────────────────────────────────────────────── @@ -1816,12 +1881,14 @@ async def status(self, session_id: str = "") -> dict[str, Any]: "context_budget_snapshot": context_metadata.get("context_budget_snapshot", {}), "session_id": str(getattr(engine, "_current_session_id", "") or "") if engine is not None else "", "runtime_source": runtime_source(), + "runtime_installation_error": await asyncio.to_thread(runtime_installation_error), "runtime_executable": sys.executable, "runtime_version": runtime_version(), "pending_approvals": self._approval_coordinator.pending_count(), "turn_admission": self._turn_admission_status(), "deferred_init": self._deferred_init_status(ctx), "watch_summary": await self._monitor_coordinator.get_summary(), + "notification_delivery": self.notification_bus.stats, "host_backend": host, "environment_sources": { "active": list(self._environment_source_manager.source_ids) diff --git a/src/leapflow/dashboard/static/app.js b/src/leapflow/dashboard/static/app.js index 25c52141..6b3f4154 100644 --- a/src/leapflow/dashboard/static/app.js +++ b/src/leapflow/dashboard/static/app.js @@ -827,6 +827,7 @@ function asArray(value) { return Array.isArray(value) ? value : []; } + const SEVERITIES = ["critical", "alert", "notable", "info"]; function severityOf(item) { return String((item && item.severity) || "info").toLowerCase(); } function severityCounts(items) { @@ -1167,7 +1168,7 @@ } else { severity = true; const counts = severityCounts(data); - rows = ["alert", "notable", "info"].map((key) => ({ key, label: t(key), value: counts[key] || 0 })); + rows = SEVERITIES.map((key) => ({ key, label: t(key), value: counts[key] || 0 })); } const max = Math.max(1, ...rows.map((r) => r.value)); rows.forEach((row) => { @@ -1266,12 +1267,18 @@ } function renderPie(data, title) { - const counts = severityCounts(data); const total = Math.max(1, (counts.alert || 0) + (counts.notable || 0) + (counts.info || 0)); + const counts = severityCounts(data); + const total = Math.max(1, SEVERITIES.reduce((sum, key) => sum + (counts[key] || 0), 0)); const d = el("div", "chart card pie-card"); if (title) d.appendChild(el("div", "card-title", esc(tx(title)))); const pie = el("div", "pie"); - pie.style.background = "conic-gradient(var(--alert) 0 " + ((counts.alert || 0) / total * 100) + "%, var(--notable) 0 " + (((counts.alert || 0) + (counts.notable || 0)) / total * 100) + "%, var(--info) 0 100%)"; - d.appendChild(pie); d.appendChild(renderLegend(["alert", "notable", "info"], counts)); return d; + let cumulative = 0; + pie.style.background = "conic-gradient(" + SEVERITIES.map((key) => { + const start = cumulative / total * 100; + cumulative += counts[key] || 0; + return "var(--" + key + ") " + start + "% " + (cumulative / total * 100) + "%"; + }).join(", ") + ")"; + d.appendChild(pie); d.appendChild(renderLegend(SEVERITIES, counts)); return d; } function renderLegend(keys, counts) { @@ -2038,13 +2045,32 @@ } } + let criticalToast = null; + let criticalToastTimer = null; + let monitorRefreshTimer = null; + + function scheduleMonitorRefresh() { + if (monitorRefreshTimer !== null) return; + monitorRefreshTimer = setTimeout(() => { + monitorRefreshTimer = null; + fetchView(); + }, 250); + } + function toast(finding) { - const sev = (finding.severity || "info").toLowerCase(); - const t = el("div", "toast sev-" + sev); - t.appendChild(el("div", "card-title", esc(finding.title))); - if (finding.summary) t.appendChild(el("div", "summary", esc(finding.summary))); - toastsEl.appendChild(t); - setTimeout(() => t.remove(), 8000); + if (finding.severity !== "critical") return; + if (criticalToastTimer !== null) clearTimeout(criticalToastTimer); + if (criticalToast) criticalToast.remove(); + criticalToast = el("div", "toast sev-critical"); + criticalToast.setAttribute("role", "status"); + criticalToast.appendChild(el("div", "card-title", esc(String(finding.title || "Critical condition").slice(0, 256)))); + criticalToast.appendChild(el("div", "summary", esc("Inspect current state and stored evidence before continuing affected work."))); + toastsEl.appendChild(criticalToast); + criticalToastTimer = setTimeout(() => { + criticalToast.remove(); + criticalToast = null; + criticalToastTimer = null; + }, 8000); } // ── Live updates over WebSocket ── @@ -2058,7 +2084,8 @@ ws.onclose = () => { setConnectionStatus("reconnecting…"); setTimeout(connectWS, 3000); }; ws.onmessage = (ev) => { let msg; try { msg = JSON.parse(ev.data); } catch (_) { return; } - if (msg.type === "monitor.finding") { toast(msg.payload || {}); fetchView(); } + if (msg.type === "monitor.finding") { toast(msg.payload || {}); scheduleMonitorRefresh(); } + else if (msg.type === "monitor.updated") { scheduleMonitorRefresh(); } else if (msg.type === "approval_request") { showApproval(msg.payload || {}); } else if (msg.type === "watch.state") { fetchView(); } else if (msg.type === "evolution.presentation") { updateEvolutionLive(msg.payload || {}); } diff --git a/src/leapflow/dashboard/static/index.html b/src/leapflow/dashboard/static/index.html index 17b4b38f..6c231c76 100644 --- a/src/leapflow/dashboard/static/index.html +++ b/src/leapflow/dashboard/static/index.html @@ -7,7 +7,7 @@ - +