Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
c3a2a1a
merge
wangxingjun778 Oct 13, 2026
950b941
ok Merge branch 'feat/mhs' of github.com:modelscope/leapflow into fea…
wangxingjun778 Oct 15, 2026
acd9464
ok Merge branch 'feat/mhs' of github.com:modelscope/leapflow into fea…
wangxingjun778 Oct 15, 2026
78a2563
remove unused content
wangxingjun778 Oct 16, 2026
6a5aea3
refactor and enhance hardware framework
wangxingjun778 Oct 19, 2026
e5b133d
add enhance hardware skills
wangxingjun778 Oct 20, 2026
1093d18
ok Merge branch 'feat/mhs' of github.com:modelscope/leapflow into fea…
wangxingjun778 Oct 20, 2026
ade894a
refactor hardware drivers
wangxingjun778 Oct 22, 2026
6da5b04
bump version
wangxingjun778 Oct 22, 2026
cb8866b
refine hardware skills and support for new sdk for piper-x
wangxingjun778 Oct 23, 2026
9186fea
refactor and refine hardware for specific sys topology
wangxingjun778 Oct 24, 2026
8a8d51e
refactor & enhance hardware module and add multi-source for media sig…
wangxingjun778 Oct 26, 2026
42dcf8a
feat(vision): share camera capture and harden preview lifecycle
wangxingjun778 Oct 27, 2026
304d5fc
add new module: LeapCalib
wangxingjun778 Oct 27, 2026
30740ec
enhance LeapCalib
wangxingjun778 Oct 30, 2026
9191107
chore: normalize author identity via .mailmap
wangxingjun778 Nov 5, 2026
0bcc6ac
ok Merge branch 'feat/mhs' of github.com:modelscope/leapflow into fea…
wangxingjun778 Nov 5, 2026
e1c1170
chore(repo): ignore .qoder and normalize mislabeled author identity
wangxingjun778 Nov 5, 2026
d11b604
chore(repo): drop obsolete mailmap entry for the rewritten identity
wangxingjun778 Nov 5, 2026
8cd639c
refactor leapcalib
wangxingjun778 Nov 12, 2026
d20bde2
fix leapd arm halt issue
wangxingjun778 Nov 12, 2026
faf1c17
grasp box success
wangxingjun778 Nov 12, 2026
0916784
fix worker leakage
wangxingjun778 Nov 13, 2026
dbb8eef
add leapboard stop and manage
wangxingjun778 Nov 13, 2026
2c5420c
add plan-time reach pre-check to motion.sequence
wangxingjun778 Nov 13, 2026
36104c7
refine pipeline for NL control devices
wangxingjun778 Nov 13, 2026
86404e5
fix hang issue and real jetson board deployment
wangxingjun778 Nov 16, 2026
2669bb8
fix tui terminal crash
wangxingjun778 Nov 16, 2026
577426a
fix vision capabilities
wangxingjun778 Nov 16, 2026
32281f8
ok Merge branch 'main' of github.com:modelscope/leapflow into feat/mhs
wangxingjun778 Nov 16, 2026
524841a
bump version
wangxingjun778 Nov 16, 2026
6f87152
fix hw_skill_run args issue
wangxingjun778 Nov 16, 2026
5eaffb1
fix piper-x halt policy and add hold-in-place policy
wangxingjun778 Nov 17, 2026
5857393
fix tui crash; enhance grasp async-batch perception
wangxingjun778 Nov 17, 2026
55c4408
fix config health classification and guard live preview lifecycle
wangxingjun778 Nov 17, 2026
ebef553
fix candidates and leapboard monitor
wangxingjun778 Nov 17, 2026
33090e4
ok Merge branch 'main' of github.com:modelscope/leapflow into feat/mhs
wangxingjun778 Nov 18, 2026
bfe94fd
bump version
wangxingjun778 Nov 18, 2026
b50bb59
feat: licensed hardware observation policy and graceful node stop
wangxingjun778 Nov 18, 2026
5569933
ok Merge branch 'main' of github.com:modelscope/leapflow into feat/mhs
wangxingjun778 Nov 18, 2026
a95f2be
bump version
wangxingjun778 Nov 18, 2026
5c08f60
fix: supervise unregistered fleet nodes, explain leapd shutdowns, blo…
wangxingjun778 Nov 19, 2026
5ab1186
fix(daemon): separate backend RPC failures from connection loss, refu…
wangxingjun778 Nov 19, 2026
66b523c
feat(monitor): bounded critical-only delivery with durable incident i…
wangxingjun778 Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ The host sends **goals**; the node decomposes them locally. A skill is planned o
| Act | `hw_configure` · `hw_actuate` · `hw_dispense` · `hw_batch_actuate` | approval-gated |
| Safety | `hw_estop` | no approval — node-local; holds in place, reports `stop_effect` / `stop_verified` |
| Control | `hw_control_start` · `hw_control_stop` · `hw_control_status` | leased deadman |
| Fleet | `hw_fleet_status` · `hw_fleet_discover` · `hw_fleet_halt_all` | multi-node |
| Fleet | `hw_fleet_status` · `hw_fleet_discover` · `hw_fleet_reconnect` · `hw_fleet_halt_all` | multi-node; unreachable nodes auto-retry |
| Skills | `hw_skill_list` · `hw_skill_run` · `hw_skill_status` · `hw_skill_stop` | plan-digest approval, node-local |
| Data | `hw_skill_label` | labels a recorded episode's outcome |

Expand Down
7 changes: 7 additions & 0 deletions src/leapflow/cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,13 @@ def main(argv: list[str] | None = None) -> int:
)
hw_validate.add_argument("--devices", default="", metavar="DIR", help="Device YAML for --workcells")
hw_sub.add_parser("permit-key", parents=[hw_json], help="Generate an Ed25519 keypair for execution permits")
hw_fleet = hw_sub.add_parser("fleet", help="Inspect and supervise configured LHP fleet nodes (via leapd)")
hw_fleet_sub = hw_fleet.add_subparsers(dest="fleet_action")
hw_fleet_reconnect = hw_fleet_sub.add_parser(
"reconnect", parents=[hw_json],
help="Retry registering unregistered fleet nodes now (no leapd restart needed)",
)
hw_fleet_reconnect.add_argument("node", nargs="?", default="", help="Configured node_id; omit for all")
# Onboarding subcommands
hw_init = hw_sub.add_parser("init", parents=[hw_json], help="Generate an annotated hc.v1 device template")
hw_init.add_argument("device_id", help="Device id for the new template")
Expand Down
113 changes: 107 additions & 6 deletions src/leapflow/cli/commands/daemon.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,14 @@ async def _run() -> int:
return asyncio.run(_run())


def _stop(runtime_dir: Path, *, force: bool = False, timeout_s: float = 10.0) -> int:
def _stop(
runtime_dir: Path,
*,
force: bool = False,
timeout_s: float = 10.0,
action: str = "stop",
guarded: bool = False,
) -> int:
from leapflow.daemon.lifecycle import DaemonInfo, cleanup_stale, stop_daemon

info = DaemonInfo.discover(runtime_dir)
Expand All @@ -208,9 +215,14 @@ def _stop(runtime_dir: Path, *, force: bool = False, timeout_s: float = 10.0) ->
print("leapd is not running.")
return 0

if not guarded:
refusal = _stop_guard(info, action=action, force=force)
if refusal is not None:
return refusal

graceful_requested = False
if info.is_healthy and info.sock_path is not None:
graceful_requested = _request_shutdown(info.sock_path)
graceful_requested = _request_shutdown(info.sock_path, action=action)
print(f"Stopping leapd (pid={info.pid})...")
result = stop_daemon(
runtime_dir,
Expand All @@ -230,12 +242,90 @@ def _stop(runtime_dir: Path, *, force: bool = False, timeout_s: float = 10.0) ->
return 1


def _request_shutdown(sock_path: Path) -> bool:
_SELF_TERMINATION_EXIT = 3
HOST_DAEMON_PID_ENV = "LEAPFLOW_HOST_DAEMON_PID"


def _stop_guard(info: object, *, action: str, force: bool) -> int | None:
"""Refuse a stop/restart that would kill a turn; return the exit code or None.

Two layers. A process spawned by the very leapd it targets (an agent's
``shell_run``, terminal session, MCP server -- all inherit
``LEAPFLOW_HOST_DAEMON_PID``) is refused unconditionally, ``--force``
included: the turn issuing the command runs inside that daemon and would
be torn down mid-flight, which is exactly how a detached ``nohup ... leap
daemon restart`` left the TUI with an unexplained dropped connection.
Otherwise, a daemon with active turns needs confirmation on a TTY and
``--force`` elsewhere, since stopping it interrupts those turns.
"""
pid = getattr(info, "pid", None)
marker = os.environ.get(HOST_DAEMON_PID_ENV, "").strip()
if pid is not None and marker == str(pid):
sys.stderr.write(
f"Refusing 'leap daemon {action}': this command was launched from inside "
f"leapd (pid={pid}) itself, so it would terminate the agent turn that "
"issued it (--force does not override this).\n"
"- An unreachable fleet node is retried automatically; use "
"hw_fleet_reconnect (or 'leap hw fleet reconnect') to retry now.\n"
"- A restart-required setting: finish the turn and ask the user to run "
f"'leap daemon {action}' from their own terminal.\n"
)
return _SELF_TERMINATION_EXIT
if force:
return None
active = _active_turns(info)
if not active:
return None
ids = ", ".join(active)
if sys.stdin is not None and sys.stdin.isatty():
try:
answer = input(
f"leapd has {len(active)} active turn(s) ({ids}); '{action}' will "
"interrupt them and they may leave partial side effects. Continue? [y/N] "
)
except EOFError:
answer = ""
if answer.strip().lower() in {"y", "yes"}:
return None
print(f"'leap daemon {action}' cancelled.")
return 1
sys.stderr.write(
f"Refusing 'leap daemon {action}': leapd has {len(active)} active turn(s) "
f"({ids}) that would be interrupted. Wait for them to finish, cancel them "
"with /cancel, or pass --force to interrupt them.\n"
)
return 1


def _active_turns(info: object) -> list[str]:
"""Return active turn request ids, or [] when leapd cannot be asked."""
sock_path = getattr(info, "sock_path", None)
if not getattr(info, "is_healthy", False) or sock_path is None:
return []
try:
status = asyncio.run(_runtime_status(sock_path))
except Exception: # noqa: BLE001 - an unresponsive daemon must stay stoppable
return []
admission = status.get("turn_admission") if isinstance(status, dict) else None
if not isinstance(admission, dict):
return []
ids = [str(item) for item in admission.get("active_request_ids") or []]
count = int(admission.get("active", 0) or 0)
if count and not ids:
ids = [f"{count} unnamed"]
return ids


def _request_shutdown(sock_path: Path, *, action: str = "stop") -> bool:
from leapflow.daemon.client import DaemonClient, DaemonUnavailableError

requested_by = {"pid": os.getpid(), "command": f"leap daemon {action}"}

async def _run() -> bool:
try:
await DaemonClient(sock_path, timeout_s=2.0).shutdown()
await DaemonClient(sock_path, timeout_s=2.0).shutdown(
reason=f"leap daemon {action}", requested_by=requested_by
)
return True
except DaemonUnavailableError:
return False
Expand All @@ -244,20 +334,31 @@ async def _run() -> bool:


def _restart(settings: object, mock_host: bool, *, force: bool = False) -> int:
from leapflow.daemon.lifecycle import DaemonInfo

runtime_dir = settings.runtime_dir
info = DaemonInfo.discover(runtime_dir)
if info.is_running:
refusal = _stop_guard(info, action="restart", force=force)
if refusal is not None:
return refusal
print("Restarting leapd...")
stop_code = _stop(runtime_dir, force=force, timeout_s=10.0)
stop_code = _stop(runtime_dir, force=force, timeout_s=10.0, action="restart", guarded=True)
if stop_code != 0:
sys.stderr.write("Restart aborted because old leapd did not stop cleanly.\n")
return stop_code
return _start(settings, mock_host)



async def _serve(settings: object, mock_host: bool) -> int:
from leapflow.daemon.server import serve_daemon
from leapflow.logging_setup import init_daemon_logging

# Every child leapd spawns (shell_run, terminal sessions, MCP servers, and
# anything they detach with nohup/setsid) inherits this marker, so a
# ``leap daemon stop|restart`` issued from inside a turn can recognise that
# it would kill its own host and refuse.
os.environ[HOST_DAEMON_PID_ENV] = str(os.getpid())
# The daemon writes stdout/stderr to leapd.log; without an explicit logging
# setup Python's lastResort handler only emits WARNING+, hiding the INFO
# field evidence (deferred init progress, turn usage). daemon.log_level is
Expand Down
74 changes: 74 additions & 0 deletions src/leapflow/cli/commands/hardware.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ async def _dispatch(args: argparse.Namespace, action: str) -> int:
return await _run_verification(action, args, json_mode)
if action == "permit-key":
return _run_permit_key(args, json_mode)
if action == "fleet":
return await _run_fleet(args, json_mode)
if action in _ONBOARDING_ACTIONS:
return await _run_onboarding(action, args, json_mode)
_fail(f"Unknown hw action: {action}")
Expand Down Expand Up @@ -271,6 +273,18 @@ async def _register_fleet_read_plane(registry: Any) -> None:

async def _run_local(action: str, args: argparse.Namespace, json_mode: bool) -> int:
settings = load_config()
if action == "list":
from leapflow.daemon.lifecycle import DaemonInfo

info = DaemonInfo.discover(settings.runtime_dir)
if info.is_running:
return _emit_result(action, {
"ok": False, "code": "hardware_store_owned_by_daemon",
"failure_code": "hardware_store_owned_by_daemon",
"failure_class": "resource_busy", "side_effect_state": "none",
"error": f"leapd (pid={info.pid}) owns the hardware database; local reconciliation is unavailable.",
"next_step": "Use hw_list in the TUI or `leap daemon status`; do not restart leapd inside a turn.",
}, json_mode)
registry = _build_local_registry(settings)
if registry is None:
return _emit_result(
Expand Down Expand Up @@ -374,6 +388,41 @@ async def _run_sampling_control(
return _emit_result(action, result, json_mode)


async def _run_fleet(args: argparse.Namespace, json_mode: bool) -> int:
"""``leap hw fleet reconnect [node]`` -- retry node registration inside leapd.

Fleet supervision lives only in leapd, so this always routes through its RPC;
retrying in this process would register the node into a throwaway registry.
"""
fleet_action = getattr(args, "fleet_action", None)
if fleet_action != "reconnect":
_fail("Usage: leap hw fleet reconnect [node]")
return 1
node = str(getattr(args, "node", "") or "").strip()
action = "fleet reconnect"
info = _discover_daemon(load_config())
if info is None:
return _emit_result(
action,
{
"ok": False,
"code": "daemon_required",
"error": (
"Fleet nodes are supervised by leapd, which is not running. Start it with "
"`leap daemon start`; configured nodes register on startup and are "
"retried automatically while unreachable."
),
},
json_mode,
)
client = DaemonClient(info.sock_path)
try:
result = await client.hardware_fleet_reconnect(node)
except DaemonUnavailableError as exc:
result = {"ok": False, "code": "daemon_error", "error": f"leapd request failed: {exc}"}
return _emit_result(action, result, json_mode)


# ── Output ─────────────────────────────────────────────────────────────────


Expand Down Expand Up @@ -417,6 +466,8 @@ def _render(action: str, result: dict[str, Any]) -> None:
print(f"{_CYAN}LEAP Hardware \u2014 {action}{_RESET}")
if not result.get("ok"):
_fail(str(result.get("error") or result.get("code") or "command failed"))
if result.get("next_step"):
_info(str(result["next_step"]))
return
renderer = {
"list": _render_list,
Expand All @@ -427,11 +478,34 @@ def _render(action: str, result: dict[str, Any]) -> None:
"pause": _render_sampling,
"resume": _render_sampling,
"replay": _render_replay,
"fleet reconnect": _render_fleet_reconnect,
}.get(action)
if renderer is not None:
renderer(result)


def _render_fleet_reconnect(result: dict[str, Any]) -> None:
nodes = result.get("nodes") or {}
if not nodes:
_info("No unregistered fleet nodes; every configured node is registered.")
return
for node_id, outcome in nodes.items():
status = str(outcome.get("status") or "")
if status == "online":
devices = ", ".join(outcome.get("device_ids") or []) or "(none)"
_ok(f"{node_id} online — devices: {devices}")
elif outcome.get("retry_in_flight"):
_info(f"{node_id}: a retry is already in progress")
elif status == "unregistered":
_warn(
f"{node_id} still unreachable: {outcome.get('last_error')} "
f"(attempt {outcome.get('attempts')}, next automatic retry in "
f"{outcome.get('next_retry_in_s')}s)"
)
else:
_info(f"{node_id}: {status or outcome}")


def _render_list(result: dict[str, Any]) -> None:
devices = result.get("devices") or []
if not devices:
Expand Down
39 changes: 33 additions & 6 deletions src/leapflow/cli/commands/interactive.py
Original file line number Diff line number Diff line change
Expand Up @@ -1234,6 +1234,8 @@ async def _print_daemon_status() -> None:
runtime_source = daemon_status.get("runtime_source")
if runtime_source:
console.system(f"Runtime source: {runtime_source}")
if daemon_status.get("runtime_installation_error"):
console.warning(str(daemon_status["runtime_installation_error"]))
runtime_executable = daemon_status.get("runtime_executable")
if runtime_executable:
console.system(f"Python: {runtime_executable}")
Expand Down Expand Up @@ -1274,7 +1276,7 @@ async def _stream_response(
record_user: bool = True,
) -> None:
nonlocal turn_count, active_turn_request_id
from leapflow.daemon.client import DaemonUnavailableError
from leapflow.daemon.client import DaemonRpcError, DaemonShutdownError, DaemonUnavailableError

if record_user:
exit_stats.record_user_message()
Expand Down Expand Up @@ -1332,12 +1334,28 @@ async def _stream_response(
if not metadata.get("heartbeat"):
console.system(event.content)
turn_completed = True
except DaemonRpcError:
console.warning(
"leapd returned a request error, not a connection loss. "
"The command was not replayed; inspect the error before trying again."
)
raise
except DaemonUnavailableError as exc:
if isinstance(exc, DaemonShutdownError):
# A deliberate stop is not a transport fault: replaying the turn would
# restart a daemon the user just stopped and could repeat side effects.
console.warning(
f"{exc.describe()}; this turn was interrupted and may have left "
"partial side effects. Check what it changed, then re-issue the "
"instruction (the TUI reconnects to leapd and keeps this session)."
)
raise
if allow_retry and not saw_real_event:
retry_error = exc
else:
progress = "after receiving turn output" if saw_real_event else "again before output"
console.warning(
"Lost connection to leapd after the turn started; "
f"Lost connection to leapd {progress}; "
"the command was not replayed to avoid duplicate side effects."
)
raise
Expand Down Expand Up @@ -1639,7 +1657,7 @@ def _handle_task_control(text: str) -> bool:
async def _notification_listener() -> None:
"""Subscribe to daemon notifications and render them in TUI."""
nonlocal daemon_session_mode
from leapflow.daemon.client import DaemonUnavailableError
from leapflow.daemon.client import DaemonShutdownError, DaemonUnavailableError

async def _refresh_watch_count() -> None:
try:
Expand Down Expand Up @@ -1691,17 +1709,26 @@ async def _refresh_watch_count() -> None:
console.system("Distillation complete — no skills produced (insufficient signal)")
app.invalidate()
elif event_type == "monitor.finding":
console.monitor_card(payload)
if str(payload.get("severity")) == "alert":
# Defense in depth for notifications from older daemons.
if payload.get("severity") == "critical" and console.monitor_notice(payload):
status.update_monitor_counts(alerts=status.alert_count + 1)
app.invalidate()
app.invalidate()
elif event_type == "watch.state":
await _refresh_watch_count()
elif event_type == "signal.stream":
# Track signal flow activity for health visibility
status.increment_signal_stream()
elif event_type == "monitor.error":
logger.debug("monitor error notification: %s", payload)
except DaemonShutdownError as exc:
# A running turn reports this itself; an idle TUI would otherwise
# only find out on its next command.
if not app.agent_running:
console.warning(
f"{exc.describe()}. The next command reconnects to leapd "
"and keeps this session."
)
await asyncio.sleep(3.0)
except (DaemonUnavailableError, OSError, asyncio.IncompleteReadError):
await asyncio.sleep(3.0)
except asyncio.CancelledError:
Expand Down
Loading
Loading