diff --git a/README.md b/README.md index ef2064f8..78b519f4 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,42 @@ export LEAPFLOW_FLEET_NODES='[{"node_id":"edge01","endpoint":"tcp://:8767"," leap daemon restart && leap hw list ``` +**Stop the edge node** (run on the node, with the same user and profile): + +```bash +leap hw stop # stop the sole local node and its workers +leap hw stop --node-id edge01 # select a node if several are running +leap hw stop --timeout 60 --json # wait longer; structured outcome +``` + +`stop` uses the owning systemd user unit when present, preventing its restart +policy from immediately relaunching the node; otherwise it sends SIGTERM to the +identified `serve` process. It waits for the node and observed workers to exit. +Repeated stops return `not_running`; ambiguous targets are refused. Discovery is +local and profile-scoped, and the host daemon/TUI and unrelated services remain +running. On Linux, legacy nodes without lifecycle records can also be discovered; +on macOS, the node must have been started by a version that writes these records. +A timeout returns failure without sending SIGKILL. A systemd unit can independently +force termination at its own `TimeoutStopSec`; an abnormal unit result is reported +as unclean, not as a successful graceful stop. + +Stopping the service is **not hardware power-off or a parking operation**: +- Graceful shutdown cancels runs/leases, attempts declared actuator halts, and + releases camera streams, previews, and device connections. +- Piper halt attempts a powered hold-in-place; closing CAN sends no disable, + reset, parking move, or gripper-open command. An enabled arm may remain energized + and a gripper may retain its last command. Actual posture, holding torque, and + object retention after disconnect are **not verified**; other hardware follows + its driver's and firmware's shutdown behavior. +- Node-side monitoring/deadman and host remote control/estop through this node are + unavailable after exit. Keep the hardware attended and follow its physical safety + procedures. Host fleet health changes on its configured heartbeat schedule. + +The command prints these consequences and distinguishes cleanup completion from +physical verification (`hardware_state_verified: false`). For a supervised node, +resume with `systemctl --user start `; otherwise reuse the original +`leap hw serve ...` command. Stopping a unit does not disable its boot activation. +
diff --git a/src/leapflow/cli/cli.py b/src/leapflow/cli/cli.py index c4288856..fa79d380 100644 --- a/src/leapflow/cli/cli.py +++ b/src/leapflow/cli/cli.py @@ -353,6 +353,19 @@ def main(argv: list[str] | None = None) -> int: "--log-file", default="", help="Append and size-rotate the node log here (config hardware.node_log_file)" ) hw_serve.add_argument("--verbose", action="store_true", help="Log at DEBUG level") + from leapflow.hardware.lhp.node_lifecycle import DEFAULT_STOP_TIMEOUT_S + + hw_stop = hw_sub.add_parser( + "stop", parents=[hw_json], help="Gracefully stop a local LHP node and its workers", + description=( + "Stop the local hw serve node in the active profile, through its supervisor " + "when present. This is not hardware power-off: Piper holding torque/grip may " + "remain; physical state is unverified after disconnect. Node monitoring and " + "remote control stop, and camera streams are released. No automatic SIGKILL." + ), + ) + hw_stop.add_argument("--node-id", default="", help="Select a local node (env LEAPFLOW_LHP_NODE_ID); required if ambiguous") + hw_stop.add_argument("--timeout", type=float, default=DEFAULT_STOP_TIMEOUT_S, help="Seconds to wait for node, workers, and supervisor to stop (default: %(default)s)") # Verification subcommands hw_verify = hw_sub.add_parser("verify", parents=[hw_json], help="Verify a device declaration (digest-bound)") hw_verify.add_argument("device", help="Device id to verify") diff --git a/src/leapflow/cli/commands/hardware.py b/src/leapflow/cli/commands/hardware.py index 4ddb29c5..cfc94446 100644 --- a/src/leapflow/cli/commands/hardware.py +++ b/src/leapflow/cli/commands/hardware.py @@ -28,7 +28,7 @@ import json import logging import sys -from dataclasses import replace +from dataclasses import asdict, replace from pathlib import Path from typing import Any @@ -132,6 +132,8 @@ async def _dispatch(args: argparse.Namespace, action: str) -> int: return _run_offline(action, args, json_mode) if action == "serve": return await _run_serve(args) + if action == "stop": + return await _run_stop(args, json_mode) if action == "validate" and getattr(args, "workcells", None) is not None: from leapflow.cli.commands.hardware_workcells import run_workcell_validation @@ -188,6 +190,51 @@ async def _run_serve(args: argparse.Namespace) -> int: raise +async def _run_stop(args: argparse.Namespace, json_mode: bool) -> int: + """Stop only the selected local node; never initialize a second hardware owner.""" + import math + import os + + from leapflow.hardware.lhp.node_lifecycle import ( + DEFAULT_STOP_TIMEOUT_S, NODE_ID_ENV, StopResult, discover_nodes, stop_node, + ) + from leapflow.hardware.lhp.protocol import is_valid_device_id + + node_id = str(getattr(args, "node_id", "") or os.environ.get(NODE_ID_ENV, "")).strip() + timeout_s = float(getattr(args, "timeout", DEFAULT_STOP_TIMEOUT_S)) + if sys.platform == "win32": + return _emit_result("stop", StopResult( + ok=False, status="unsupported_platform", detail="Graceful node stop is supported on POSIX systems only.", + ).to_dict(), json_mode) + if (node_id and not is_valid_device_id(node_id)) or not math.isfinite(timeout_s) or timeout_s <= 0: + return _emit_result("stop", StopResult( + ok=False, status="invalid_args", detail="Use a valid --node-id and a positive, finite --timeout.", + ).to_dict(), json_mode) + try: + settings = load_config() + nodes = await asyncio.to_thread(discover_nodes, settings.runtime_dir, node_id) + except (OSError, ValueError) as exc: + logger.error("Local node discovery failed", exc_info=True) + return _emit_result("stop", StopResult( + ok=False, status="discovery_failed", detail=f"Cannot discover local nodes: {type(exc).__name__}; no stop requested.", + ).to_dict(), json_mode) + if not nodes: + return _emit_result("stop", StopResult( + ok=True, status="not_running", detail="No matching local node is running in this profile; no service action taken.", + ).to_dict(), json_mode) + if len(nodes) != 1: + candidates = ", ".join(f"{node.node_id or '(unknown id)'} (pid {node.pid})" for node in nodes) + return _emit_result("stop", StopResult( + ok=False, status="ambiguous_node", + detail=f"Multiple local nodes match: {candidates}. Select one with --node-id or the correct profile; no stop requested.", + ).to_dict(), json_mode) + if not json_mode: + print("Warning: stopping the node cancels work and disconnects hardware. This is NOT power-off; physical arm/grip state is unverified afterward.", flush=True) + print(f"Waiting up to {timeout_s:g}s for {nodes[0].node_id} and its workers to stop...", flush=True) + result = await asyncio.to_thread(stop_node, nodes[0], timeout_s=timeout_s) + return _emit_result("stop", result.to_dict(), json_mode) + + # ── Read / estop plane ───────────────────────────────────────────────────── @@ -331,6 +378,26 @@ async def _run_sampling_control( def _emit_result(action: str, result: dict[str, Any], json_mode: bool) -> int: + if action == "stop": + # FailureEnvelope belongs at the application boundary, not in hardware. + from leapflow.engine.recovery.failure_envelope import ( + FailureEnvelope, FailureSource, Recoverability, SideEffectState, + ) + + failure = None + if not result.get("ok"): + envelope = FailureEnvelope.create( + source=FailureSource.SYSTEM, category="node_lifecycle", + failure_class="NodeStopFailure", failure_code=result["status"], + message=result["detail"], recoverability=Recoverability.USER_FIXABLE, + side_effect_state=SideEffectState.UNKNOWN if result.get("via") else SideEffectState.NONE, + ) + failure = { + **asdict(envelope), "source": envelope.source.value, + "recoverability": envelope.recoverability.value, + "side_effect_state": envelope.side_effect_state.value, + } + result = {**result, "failure": failure} if json_mode: print(json.dumps(result, indent=2, default=str, ensure_ascii=False)) else: @@ -339,6 +406,14 @@ def _emit_result(action: str, result: dict[str, Any], json_mode: bool) -> int: def _render(action: str, result: dict[str, Any]) -> None: + if action == "stop": + print("LEAP Hardware — stop") + print(f" {result.get('status')}: {result.get('detail', '')}") + for consequence in result.get("consequences", ()): + print(f" Warning: {consequence}") + if result.get("survivors"): + print(f" Worker processes still running: {result['survivors']}") + return print(f"{_CYAN}LEAP Hardware \u2014 {action}{_RESET}") if not result.get("ok"): _fail(str(result.get("error") or result.get("code") or "command failed")) @@ -1007,7 +1082,7 @@ async def _run_commission(args: argparse.Namespace, json_mode: bool) -> int: def _print_usage() -> None: - print("Usage: leap hw {list|describe|read|status|estop|pause|resume|replay|serve|verify|validate|permit-key|init|probe|commission} [--json]") + print("Usage: leap hw {list|describe|read|status|estop|pause|resume|replay|serve|stop|verify|validate|permit-key|init|probe|commission} [--json]") print() print("Inspect hardware and intervene in it directly.") print() @@ -1021,6 +1096,7 @@ def _print_usage() -> None: print(" resume Resume daemon sampling for a device") print(" replay Replay a raw NDJSON segment through the event detector") print(" serve Serve this machine as an LHP fabric node (edge runtime)") + print(" stop [--node-id ID] Stop the local node and workers; not hardware power-off") print(" verify Write a digest-bound verification entry for a device") print(" validate [device] Validate declarations against their verification entries") print(" permit-key Show the permit issuer public key and key_id") diff --git a/src/leapflow/cli/commands/slash_handlers.py b/src/leapflow/cli/commands/slash_handlers.py index 0c618162..cd699fab 100644 --- a/src/leapflow/cli/commands/slash_handlers.py +++ b/src/leapflow/cli/commands/slash_handlers.py @@ -251,7 +251,7 @@ def build_config_payload(ctx: "Context", args: str = "") -> dict[str, Any]: """Execute a config command and return a serializable payload.""" from leapflow.config_service import ConfigService - service = ConfigService(ctx.settings) + service = ConfigService(ctx.settings, hardware_registry=getattr(ctx, "_hardware_registry", None)) try: from leapflow.utils.shell_lex import split_args diff --git a/src/leapflow/cli/context.py b/src/leapflow/cli/context.py index c0301bb8..09cf6cb0 100644 --- a/src/leapflow/cli/context.py +++ b/src/leapflow/cli/context.py @@ -1133,7 +1133,20 @@ def reload_runtime_config_if_changed(self, *, force: bool = False) -> bool: return False previous = self.settings - updated = self._load_runtime_settings_from_files() + try: + updated = self._load_runtime_settings_from_files() + except ValueError: + logger.warning("Invalid runtime configuration; retaining last valid settings", exc_info=True) + self._config_signature = signature + return False + registry = getattr(self, "_hardware_registry", None) + if registry is not None and updated.hardware_observation != previous.hardware_observation: + try: + registry.reconfigure_observation(updated.hardware_observation) + except ValueError: + from dataclasses import replace + logger.warning("Invalid hardware observation reload; retaining last valid policy", exc_info=True) + updated = replace(updated, hardware_observation=previous.hardware_observation) self._config_signature = self._runtime_config_signature(updated) llm_changed = ( previous.llm_api_key != updated.llm_api_key diff --git a/src/leapflow/config.py b/src/leapflow/config.py index 7871d716..d164f508 100644 --- a/src/leapflow/config.py +++ b/src/leapflow/config.py @@ -311,6 +311,8 @@ class Settings: # enforces (drive temperature, supply voltage, advisory fault flags). Breaches # are reported to the board with backoff and never gate a command. hardware_advisory_limits: str = "" + # JSON operational observation policy; never changes writes or task accuracy. + hardware_observation: str = "{}" # LHP edge-node process log (`leap hw serve`). With a file the node appends and # size-rotates it itself (max_mb x backups), so restarts keep earlier runs and # the disk stays bounded; empty writes to stderr only. Read at node start. @@ -1569,6 +1571,9 @@ def _build_settings_from_env( ViewPriorSettings.from_json(hardware_view_priors) hardware_alert_policies = os.getenv("LEAPFLOW_HARDWARE_ALERT_POLICIES", "").strip() hardware_advisory_limits = os.getenv("LEAPFLOW_HARDWARE_ADVISORY_LIMITS", "").strip() + hardware_observation = os.getenv("LEAPFLOW_HARDWARE_OBSERVATION", "{}").strip() + from leapflow.hardware.observation import ObservationSettings + ObservationSettings.parse(hardware_observation) hardware_node_log_file = os.getenv("LEAPFLOW_HARDWARE_NODE_LOG_FILE", "").strip() hardware_node_log_level = ( os.getenv("LEAPFLOW_HARDWARE_NODE_LOG_LEVEL", "INFO").strip() or "INFO" @@ -2139,6 +2144,7 @@ def _tuple_env(key: str, default: tuple) -> tuple: hardware_view_priors=hardware_view_priors, hardware_alert_policies=hardware_alert_policies, hardware_advisory_limits=hardware_advisory_limits, + hardware_observation=hardware_observation, hardware_node_log_file=hardware_node_log_file, hardware_node_log_level=hardware_node_log_level, hardware_node_log_max_mb=hardware_node_log_max_mb, diff --git a/src/leapflow/config_service.py b/src/leapflow/config_service.py index 43456bb7..caaee56f 100644 --- a/src/leapflow/config_service.py +++ b/src/leapflow/config_service.py @@ -299,6 +299,13 @@ class ConfigSnapshot: "else deletes from that table. Read when sampling starts, so a change needs a " "daemon restart." ), + "hardware.observation": ( + "JSON report-only policy with defaults and exact overrides (device_id/channel_id). " + "soft_min/soft_max and margin_floor/margin_max are in channel units; confirm_s/recover_s " + "require continuous valid evidence. Adaptive defaults off and requires margin_max. " + "Hard envelope and task accuracy are unchanged. Valid updates hot-reload; invalid " + "updates retain the last admitted policy. Channel bounds are checked at hardware admission." + ), "hardware.raw_segment_mb": ( "Size at which a raw sample file is closed and a new segment started. A finished " "segment is written once, so its indexed size and TTL are accurate and old data " @@ -823,8 +830,9 @@ def _profile_file_for_section(section: str) -> str: class ConfigService: """Read and mutate LeapFlow configuration through layout-owned paths.""" - def __init__(self, settings: Settings) -> None: + def __init__(self, settings: Settings, *, hardware_registry: Any = None) -> None: self._settings = settings + self._hardware_registry = hardware_registry self._secrets = ScopedSecretResolver(settings.layout, settings.profile_layout) def snapshot(self) -> ConfigSnapshot: @@ -886,6 +894,19 @@ def set( if spec.secret: return self._set_secret_backed_field(spec, str(value), scope=scope) coerced = _coerce_value(value, spec.value_type) + if normalized == "hardware.observation": + from leapflow.hardware.observation import ObservationSettings + policy = ObservationSettings.parse(coerced) + if self._hardware_registry is not None: + contexts = self._hardware_registry.contexts() + else: + # Offline CLI validation reads declarations only, never discovers or opens devices. + from leapflow.hardware.providers.yaml_provider import YamlContextProvider + from leapflow.hardware.registry import HardwareSettings + hardware = HardwareSettings.from_settings(self._settings) + yaml_config = next((config for kind, config in hardware.providers if kind == "yaml"), None) + contexts = YamlContextProvider(yaml_config).discover() if yaml_config else () + policy.validate_contexts(contexts) if normalized.startswith("hardware.skills_planning_"): from leapflow.hardware.skills.planning import PlanningSettings @@ -1034,7 +1055,9 @@ def _field_view(self, spec: ConfigFieldSpec) -> ConfigFieldView: def _with_metadata(spec: ConfigFieldSpec) -> ConfigFieldSpec: - if spec.section in _RESTART_REQUIRED_SECTIONS: + if spec.key == "hardware.observation": + reload_semantics = "yes" + elif spec.section in _RESTART_REQUIRED_SECTIONS: reload_semantics = "restart-required" elif spec.section in _PARTIAL_RELOAD_SECTIONS: reload_semantics = "partial" diff --git a/src/leapflow/daemon/monitor_coordinator.py b/src/leapflow/daemon/monitor_coordinator.py index 875f6841..18a19adf 100644 --- a/src/leapflow/daemon/monitor_coordinator.py +++ b/src/leapflow/daemon/monitor_coordinator.py @@ -254,7 +254,9 @@ def _register_hardware_producer(self, ctx: Any, settings: Any) -> None: from leapflow.hardware.observability import HardwareObservationProducer monitors.producers.register( - HardwareObservationProducer(lambda: getattr(ctx, "_hardware_registry", None)) + HardwareObservationProducer(lambda: getattr(ctx, "_hardware_registry", None), + finding_store=monitors.finding_store, + run_db=getattr(ctx, "_run_deferred_db", None)) ) except Exception: logger.debug("daemon: hardware observability producer unavailable", exc_info=True) diff --git a/src/leapflow/hardware/advisory.py b/src/leapflow/hardware/advisory.py index 6bb6a211..43cee018 100644 --- a/src/leapflow/hardware/advisory.py +++ b/src/leapflow/hardware/advisory.py @@ -27,6 +27,7 @@ from typing import Any, Callable, Iterable, Mapping, Sequence from leapflow.hardware.stream import EventKind, HardwareEvent +from leapflow.hardware.observation import Episode, DEFAULT_REPORT_AFTER_S, DEFAULT_REPORT_MAX_S logger = logging.getLogger(__name__) @@ -34,8 +35,6 @@ ADVISORY_CLEARED = EventKind.ADVISORY_CLEARED ADVISORY_KINDS = frozenset({ADVISORY_EXCEEDED, ADVISORY_CLEARED}) -DEFAULT_REPORT_AFTER_S = 60.0 -DEFAULT_REPORT_MAX_S = 900.0 DEFAULT_READ_TIMEOUT_S = 2.0 _WILDCARD = "*" @@ -173,46 +172,31 @@ def _children(item: Any, part: str) -> Iterable[tuple[Any, Any]]: return [] -@dataclass -class _State: - active: bool = False - since: float = 0.0 - next_report: float = 0.0 - interval: float = 0.0 - folded: int = 0 - - @dataclass class AdvisoryTracker: """Turn successive judgements into backed-off breach and recovery events.""" clock: Callable[[], float] = time.time - _states: dict[tuple[str, str, str], _State] = field(default_factory=dict) + _states: dict[tuple[str, str, str], Episode] = field(default_factory=dict) def observe(self, limit: AdvisoryLimit, value: Any) -> list[HardwareEvent]: judgement = limit.judge(value) if not judgement.known: return [] # nothing matched: not evidence either way key = (limit.device_id, limit.channel_id, limit.name) - state = self._states.setdefault(key, _State()) + state = self._states.setdefault(key, Episode()) now = float(self.clock()) - if judgement.breached: - if not state.active: - state.active, state.since, state.folded = True, now, 0 - state.interval = limit.report_after_s - state.next_report = now + state.interval - return [self._event(ADVISORY_EXCEEDED, limit, judgement, now, "")] - if now < state.next_report: - state.folded += 1 - return [] - note = f"; still breached after {now - state.since:.0f}s, {state.folded} poll(s) folded" - state.interval = min(state.interval * 2.0, limit.report_max_s) - state.next_report, state.folded = now + state.interval, 0 + folded = state.folded + phase = state.observe(breached=judgement.breached, clear=judgement.clear, now=now, + report_after_s=limit.report_after_s, report_max_s=limit.report_max_s) + if phase == "entry": + return [self._event(ADVISORY_EXCEEDED, limit, judgement, now, "")] + if phase == "reminder": + note = f"; still breached after {now - state.since:.0f}s, {folded} poll(s) folded" return [self._event(ADVISORY_EXCEEDED, limit, judgement, now, note)] - if state.active and judgement.clear: - lasted = now - state.since + if phase == "recovery": self._states.pop(key, None) - return [self._event(ADVISORY_CLEARED, limit, judgement, now, f" after {lasted:.0f}s")] + return [self._event(ADVISORY_CLEARED, limit, judgement, now, f" after {now - state.since:.0f}s")] return [] @staticmethod diff --git a/src/leapflow/hardware/alert_policy.py b/src/leapflow/hardware/alert_policy.py index 398b717b..4b3b0364 100644 --- a/src/leapflow/hardware/alert_policy.py +++ b/src/leapflow/hardware/alert_policy.py @@ -3,7 +3,7 @@ A policy is a YAML rule that maps an observed ``EventKind`` to an action. The mapping is loaded from ``hardware.alert_policies`` at startup and evaluated on -every event the stream emits. The two actions are: +every raw sample's condition evidence, before event presentation pacing. The two actions are: ``hw_estop`` Halt the device immediately via ``transport.halt()``. Exempt from approval @@ -17,7 +17,7 @@ ``require_consecutive`` prevents a single transient reading from triggering an irreversible response: the same *kind* on the same *channel* must fire at least -that many times in a row before the policy fires. The default (3) is chosen so +that many valid observations in a row before the policy fires once. The default (3) is chosen so a single spike is never actionable, matching the three-sample rule the quality degradation detector already uses in ``HardwareEventDetector``. @@ -156,13 +156,31 @@ def evaluate(self, event: Any) -> None: device_id = str(getattr(event, "device_id", "")) channel_id = str(getattr(event, "channel_id", "")) - for rule in self._rules: - if not rule.matches(kind, device_id, channel_id): - continue - count = self._tracker.hit(kind, device_id, channel_id) - if count < rule.require_consecutive: - continue - self._fire(rule, event, device_id) + from leapflow.hardware.stream import HardwareEventClass, classify_hardware_event + + if classify_hardware_event(kind) == HardwareEventClass.RECORD: + return # operational warnings cannot become an action through configuration + matching = [rule for rule in self._rules if rule.matches(kind, device_id, channel_id)] + if not matching: + return + count = self._tracker.hit(kind, device_id, channel_id) + for rule in matching: + if count == rule.require_consecutive: + self._fire(rule, event, device_id) + + def evaluate_sample(self, device_id: str, channel_id: str, + evidence: Mapping[str, Any | None]) -> None: + """Judge consecutive raw evidence before presentation pacing, once per rule. + + A healthy or unavailable sample breaks the relevant run, independently of + another condition on the same channel. Notifications are not evidence. + """ + for kind, event in evidence.items(): + if event is None: + key = self._tracker._key(kind, device_id, channel_id) + self._tracker.counts.pop(key, None) + else: + self.evaluate(event) def reset_channel(self, device_id: str, channel_id: str) -> None: """Clear consecutive counters when a channel recovers. diff --git a/src/leapflow/hardware/calibration/generations.py b/src/leapflow/hardware/calibration/generations.py index c8c51c51..e9f0dc16 100644 --- a/src/leapflow/hardware/calibration/generations.py +++ b/src/leapflow/hardware/calibration/generations.py @@ -606,6 +606,18 @@ def pending_events(self, *, limit: int = 100, device_id: str = "") -> tuple[dict return () if not connection.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_name='calib_outbox'").fetchone()[0]: raise ValueError("durable calibration outbox missing from committed schema") + # Lifecycle consumers reset runtime observation scope from these records. + # A digest is not chronology, and a rollback can decrease generation. + journal = connection.execute( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_name='calib_pointer_journal'" + ).fetchone()[0] + join = ( + " LEFT JOIN calib_pointer_journal j ON j.event_id=d.event_id " if journal else " " + ) + order = ( + "d.workcell_id,COALESCE(j.revision,0),d.created_at,o.event_id" + if journal else "d.workcell_id,d.created_at,o.event_id" + ) if device_id: if not isinstance(device_id, str) or len(device_id) > 256: raise ValueError("bounded exact lifecycle device required") @@ -614,15 +626,18 @@ def pending_events(self, *, limit: int = 100, device_id: str = "") -> tuple[dict # record-only events through their explicitly bound causal device. rows = connection.execute( "SELECT o.event_json FROM calib_outbox o JOIN calib_decisions d ON o.event_id=d.event_id " - "WHERE NOT o.delivered AND (json_extract_string(d.decision_json,'$.causal.device_id')=? " + + join + "WHERE NOT o.delivered AND (json_extract_string(d.decision_json,'$.causal.device_id')=? " "OR EXISTS (SELECT 1 FROM calib_generations g WHERE g.workcell_id=d.workcell_id " "AND g.generation=COALESCE(TRY_CAST(json_extract_string(d.decision_json,'$.generation') AS BIGINT), " "TRY_CAST(json_extract_string(d.decision_json,'$.previous_generation') AS BIGINT)) " - "AND json_extract_string(g.model_json,'$.device_id')=?)) ORDER BY o.event_id LIMIT ?", + "AND json_extract_string(g.model_json,'$.device_id')=?)) ORDER BY " + order + " LIMIT ?", [device_id, device_id, limit], ).fetchall() else: - rows = connection.execute("SELECT event_json FROM calib_outbox WHERE NOT delivered ORDER BY event_id LIMIT ?", [limit]).fetchall() + rows = connection.execute( + "SELECT o.event_json FROM calib_outbox o JOIN calib_decisions d ON o.event_id=d.event_id " + + join + "WHERE NOT o.delivered ORDER BY " + order + " LIMIT ?", [limit] + ).fetchall() return tuple(self._read_decision(row[0]) for row in rows) def acknowledge_event(self, event_id: str) -> None: diff --git a/src/leapflow/hardware/drivers/_piper_feedback.py b/src/leapflow/hardware/drivers/_piper_feedback.py index 4fb66900..bd987698 100644 --- a/src/leapflow/hardware/drivers/_piper_feedback.py +++ b/src/leapflow/hardware/drivers/_piper_feedback.py @@ -275,10 +275,15 @@ def space(self, space: str) -> list[float] | None: return read_joints(self._robot) if space == SPACE_JOINT else read_pose(self._robot) def gripper(self) -> tuple[float | None, float | None]: - """Gripper stroke (mm) and force (N), or Nones when unavailable.""" + """SDK stroke (mm) and signed force report (N), not qualified metrology. + + The SDK decodes force as signed int16 mN, then scales to N. Neither + its physical sign convention nor a zero/load calibration is certified here. + """ state = sdk_message(self._gripper.get_gripper_status) if self._gripper is not None else None stroke, force = numeric(attr(state, "value")), numeric(attr(state, "force")) - return (stroke * _MM_PER_M if stroke is not None else None, force) + mode = attr(state, "mode") + return (stroke * _MM_PER_M if stroke is not None and mode != "angle" else None, force) def poll(self, channel_ids: tuple[str, ...]) -> dict[str, Any]: """Read the requested feedback channels plus the receipts that certify them. @@ -298,7 +303,7 @@ def poll(self, channel_ids: tuple[str, ...]) -> dict[str, Any]: if not wanted.isdisjoint(GRIPPER_CHANNELS): stroke, force = self.gripper() values[GRIPPER_POSITION] = stroke - values[GRIPPER_EFFORT] = force if stroke is not None else None + values[GRIPPER_EFFORT] = force if ARM_ENABLED in wanted: enabled = enable_status(self._robot) values[ARM_ENABLED] = all(enabled) if enabled is not None else None @@ -337,6 +342,20 @@ def _telemetry(self) -> dict[str, Any]: "faults": {name: attr(foc, name) for name in _FAULT_FLAGS}, }) receipts = self.receipts() + gripper = sdk_message(self._gripper.get_gripper_status) if self._gripper is not None else None + gripper_status = attr(gripper, "foc_status") + status = status_feedback(self._robot) return {"schema_version": 1, "motors": motors, "receive": receipts, + "settled": (as_int(attr(status, "motion_status")) == MOTION_STATUS_ARRIVED + if status is not None else None), + "status": status_value(status) if status is not None else None, "receive_qualified": not self.problem(CAN_GROUPS, receipts), - "effort_source": "sdk_current_estimate"} + "effort_source": "sdk_current_estimate", + "gripper": {"sdk_value": numeric(attr(gripper, "value")), + "mode": attr(gripper, "mode"), + "signed_force_report_n": numeric(attr(gripper, "force")), + "force_source": "sdk_signed_force_report", + "force_calibration": "unqualified", + "faults": {name: attr(gripper_status, name) for name in ( + "voltage_too_low", "motor_overheating", "driver_overcurrent", + "driver_overheating", "sensor_status", "driver_error_status")}}} diff --git a/src/leapflow/hardware/drivers/_piper_motion.py b/src/leapflow/hardware/drivers/_piper_motion.py index 23bb00cd..12dc737a 100644 --- a/src/leapflow/hardware/drivers/_piper_motion.py +++ b/src/leapflow/hardware/drivers/_piper_motion.py @@ -361,6 +361,14 @@ class HoldRefused(TransportError): """ +class TargetNotSent(TransportError): + """Motion preparation failed before any target was emitted. + + Mode or speed frames may have been sent, but no posture was commanded. + This is a motion refusal, not uncertainty about target execution. + """ + + class TargetRefused(TransportError): """The firmware planned the target and refused it; the arm did not move. @@ -580,13 +588,7 @@ def send( vector[index] = float(value) snapshot = [float(value) for value in vector] # type: ignore[arg-type] try: - if validate is not None: - validate() - if ctrl_mode(robot) != CTRL_MODE_CAN or not self._speed_sent: - self._enter_can(robot, gen) - self._assert_move_mode(robot, space, gen) - if validate is not None: - validate() + self._prepare_target(robot, space, gen, validate) if space == SPACE_JOINT: self.emit_target(gen, robot.move_j, joints_to_sdk(snapshot)) elif self._pose_path == POSE_PATH_POINT: @@ -603,6 +605,23 @@ def send( self._held = {space: (epoch, list(snapshot))} return snapshot + def _prepare_target(self, robot: Any, space: str, generation: int, + validate: Callable[[], None] | None) -> None: + """Keep the pre-target failure boundary separate from target execution.""" + try: + if validate is not None: + validate() + if ctrl_mode(robot) != CTRL_MODE_CAN or not self._speed_sent: + self._enter_can(robot, generation) + self._assert_move_mode(robot, space, generation) + if validate is not None: + validate() + except (MotionPreempted, MotionHalted, TargetNotSent): + raise + except TransportError as exc: + raise TargetNotSent(f"{exc}; no motion target was sent", + failure_code=exc.failure_code) from exc + def _confirm_target(self, robot: Any, space: str, snapshot: Sequence[float]) -> None: """Read the firmware's own answer to the posture just sent. @@ -730,6 +749,7 @@ def _assert_move_mode(self, robot: Any, space: str, generation: int) -> None: "SPACE_JOINT", "SPACE_POSE", "PiperMotion", + "TargetNotSent", "as_int", "attr", "ctrl_mode", diff --git a/src/leapflow/hardware/drivers/piper.py b/src/leapflow/hardware/drivers/piper.py index 51aaefd6..35dccf41 100644 --- a/src/leapflow/hardware/drivers/piper.py +++ b/src/leapflow/hardware/drivers/piper.py @@ -64,6 +64,7 @@ import importlib import time from typing import Any, Mapping +from uuid import uuid4 from leapflow.hardware.context import HardwareContext, Quality from leapflow.hardware.drivers._piper_feedback import ( @@ -103,6 +104,7 @@ MotionHalted, MotionPreempted, PiperMotion, + TargetNotSent, TargetRefused, enable_flag, motion_space, @@ -269,6 +271,7 @@ def __init__(self, config: Mapping[str, Any] | None = None) -> None: self._context: HardwareContext | None = None self._connected = False self._sequence = 0 + self._acquisition_epoch = "" self._firmware = "" # Hold vectors for both motion spaces, seeded from joint feedback at # open so the first single-joint write holds the other five joints. @@ -289,6 +292,7 @@ async def open(self, context: HardwareContext) -> TransportStatus: if self._connected: return await self.probe() self._context = context + self._acquisition_epoch = uuid4().hex sdk = _load_sdk() try: await asyncio.to_thread(self._bring_up, sdk) @@ -586,7 +590,7 @@ async def _write_motion(self, space: str, commands: list[tuple[str, Any]], call = _VENDOR_CALL[space if space == SPACE_JOINT else self._motion.pose_path] try: snapshot = await asyncio.to_thread(self._send_motion, robot, space, targets, generation) - except (HoldRefused, TargetRefused, MotionPreempted, MotionHalted) as exc: + except (HoldRefused, TargetNotSent, TargetRefused, MotionPreempted, MotionHalted) as exc: return [_refused(str(exc), exc.failure_code)] * len(commands) except TransportError as exc: return [_failed(SIDE_EFFECT_UNKNOWN, f"{call}: {exc}", exc.failure_code)] * len(commands) @@ -717,7 +721,10 @@ async def _wait_receive(self) -> str: await asyncio.sleep(min(0.02, max(0.0, deadline - time.monotonic()))) def _require_receive(self) -> None: - problem = self._receive_problem() + """Requalify on the motion worker too; transient skew can arise after admission.""" + feedback = self._feedback + problem = (feedback.wait_qualified(self._spec.requalify_s) if feedback is not None + else "Piper transport is not open") if problem: raise TransportError(problem, failure_code="piper_unqualified_feedback") @@ -764,6 +771,9 @@ def _reading( monotonic_at=monotonic_at, sequence=self._sequence, quality=quality, + sequence_scope="device_read", + acquisition_at=monotonic_at if receipts is not None and quality == Quality.OK.value else None, + acquisition_epoch=self._acquisition_epoch, ) def _known_channel(self, channel_id: str) -> bool: diff --git a/src/leapflow/hardware/drivers/realsense.py b/src/leapflow/hardware/drivers/realsense.py index 87599c00..fde98bb9 100644 --- a/src/leapflow/hardware/drivers/realsense.py +++ b/src/leapflow/hardware/drivers/realsense.py @@ -291,6 +291,7 @@ async def read(self, channel_id: str) -> Reading: value=value, quantity="camera_metadata", sequence=self._sequence, + sequence_scope="device_acquisition", quality=Quality.OK.value, ) @@ -313,7 +314,7 @@ async def _read_measurement(self, channel_id: str) -> Reading: return Reading(device_id=self._device_id, channel_id=channel_id, value=evidence.to_dict(), quantity="geometry_proposal" if geometry else "depth_quality", observed_at=frame.observed_at, monotonic_at=frame.monotonic_at, - sequence=frame.sequence, + sequence=frame.sequence, sequence_scope="device_acquisition", quality=Quality.OK.value if qualified else Quality.STALE.value) async def read_frame( diff --git a/src/leapflow/hardware/lhp/control_sessions.py b/src/leapflow/hardware/lhp/control_sessions.py index 476a0e12..a4acbba0 100644 --- a/src/leapflow/hardware/lhp/control_sessions.py +++ b/src/leapflow/hardware/lhp/control_sessions.py @@ -50,6 +50,9 @@ FINISH_POLL_S = 0.25 """Upper bound on how long a finished body keeps its lease bookkeeping alive.""" VALID_COMM_LOSS_POLICIES = frozenset({"halt", "hold"}) +_EFFECT_DISPATCH = "dispatch" +_EFFECT_CONTROL = "control" +_EFFECT_TRANSACTION = "transaction" # ── Session body contract ───────────────────────────────────────────────── @@ -282,7 +285,8 @@ def __init__(self, registry: Any, *, device_busy: Callable[[str], bool] | None = self._sessions: dict[str, LeasedSession] = {} self._reservations: dict[str, tuple[str, ...]] = {} self._dispatches: dict[str, str] = {} - self._uncertain_devices: set[str] = set() + # Per-source stamps keep a halt from clearing newer or non-physical failures. + self._unsettled: dict[tuple[str, str], tuple[float, str]] = {} self._reserved_tasks: set[asyncio.Task[Any]] = set() self._closing = False @@ -322,10 +326,45 @@ def unsettled_device(self, device_id: str) -> bool: return ( self._closing or self.reserved_device(device_id) or device_id in self._dispatches.values() - or device_id in self._uncertain_devices + or any(device == device_id for device, _ in self._unsettled) or any(device == device_id for device, _ in self._controls.values()) ) + def busy_message(self, device_id: str) -> str: + """Distinguish an active owner from uncertainty, with the appropriate remedy.""" + with self._mutex: + reasons = [ + f"unsettled {source} effect ({detail})" + for (device, source), (_, detail) in self._unsettled.items() + if device == device_id + ] + if not reasons: + return (f"device/consumer {device_id!r} is busy with a session, control loop, " + "transaction or unsettled dispatch; stop its owner or wait for it to end") + remedy = ("owner reconciliation is required; a halt cannot settle a model transaction" + if (device_id, _EFFECT_TRANSACTION) in self._unsettled + else "halt with hw_estop and confirm stop_verified=true before resuming") + return f"device/consumer {device_id!r} is busy: {'; '.join(reasons)}; {remedy}" + + def _mark_unsettled(self, devices: Iterable[str], source: str, detail: str) -> None: + """Record uncertainty under the owner mutex; never release it on a timer.""" + for device in devices: + self._unsettled[device, source] = (time.monotonic(), detail) + logger.warning("device %s unsettled %s effect: %s", device, source, detail) + + def settle_halt(self, device_id: str, *, before: float, evidence: str) -> bool: + """Settle physical uncertainty predating a verified halt, never persistence.""" + with self._mutex: + keys = [key for key, (stamp, _) in self._unsettled.items() + if key[0] == device_id and key[1] in (_EFFECT_DISPATCH, _EFFECT_CONTROL) + and stamp <= before] + for key in keys: + del self._unsettled[key] + if keys: + logger.warning("device %s settled physical uncertainty after verified halt: %s", + device_id, evidence) + return bool(keys) + def reserve(self, device_ids: Iterable[str], *, owner_id: str, run_id: str = "") -> str: """Atomically reserve devices/consumer IDs before any async transaction work. @@ -339,7 +378,7 @@ def reserve(self, device_ids: Iterable[str], *, owner_id: str, run_id: str = "") if run_id: self._check_transaction_run(devices, run_id) elif busy := self.busy_device(devices): - raise ValueError(f"device/consumer {busy!r} is busy or has unsettled effects") + raise ValueError(self.busy_message(busy)) token = str(uuid.uuid4()) self._reservations[token] = devices return token @@ -435,7 +474,8 @@ async def _settle_reserved( # A callable can commit and then fail before returning. An exception # alone cannot establish that persistence left the prior state intact. with self._mutex: - self._uncertain_devices.update(self._reservations[token]) + self._mark_unsettled(self._reservations[token], _EFFECT_TRANSACTION, + "persistence failed without a settlement verdict") raise # Publish the committed owner snapshot on this loop BEFORE releasing # resources, including when the original requesting lease has ended. @@ -450,7 +490,8 @@ async def _settle_reserved( # Persistence landed but its safety projection did not. Keep # every affected consumer closed until owner reconciliation. with self._mutex: - self._uncertain_devices.update(self._reservations[token]) + self._mark_unsettled(self._reservations[token], _EFFECT_TRANSACTION, + "committed safety projection was not published") raise return result finally: @@ -476,19 +517,20 @@ def begin_dispatch(self, device_id: str) -> str: with self._mutex: if ( self._closing or self.reserved_device(device_id) - or device_id in self._uncertain_devices + or any(device == device_id for device, _ in self._unsettled) or any(device == device_id for device, _ in self._controls.values()) ): - raise ValueError("device reserved for model transaction, control or uncertain prior action") + raise ValueError(self.busy_message(device_id)) token = str(uuid.uuid4()) self._dispatches[token] = device_id return token - def finish_dispatch(self, token: str, *, settled: bool) -> None: + def finish_dispatch(self, token: str, *, settled: bool, detail: str = "") -> None: with self._mutex: device = self._dispatches.pop(token) if not settled: - self._uncertain_devices.add(device) + self._mark_unsettled((device,), _EFFECT_DISPATCH, + detail or "dispatch ended without a settlement verdict") def begin_control( self, device_id: str, *, stop: Callable[[], None], reservation_token: str | None = None, @@ -506,7 +548,7 @@ def begin_control( if self._closing or self._reservations.get(reservation_token) != (device_id,): raise ValueError("exact owner control-start reservation required") elif self.busy_device((device_id,)): - raise ValueError("device is busy or has unsettled effects") + raise ValueError(self.busy_message(device_id)) token = str(uuid.uuid4()) self._controls[token] = (device_id, stop) if reservation_token is not None: @@ -518,7 +560,7 @@ def finish_control(self, token: str, *, settled: bool) -> None: with self._mutex: device, _ = self._controls.pop(token) if not settled: - self._uncertain_devices.add(device) + self._mark_unsettled((device,), _EFFECT_CONTROL, "control loop halt was not verified") # ── Lifecycle ── @@ -574,7 +616,7 @@ async def open( if context is None: return _error("unknown_device", f"unknown device {device_id!r}") if self.busy_device((device_id,)): - return _error(FAILURE_DEVICE_BUSY, f"device {device_id!r} is held by another session") + return _error(FAILURE_DEVICE_BUSY, self.busy_message(device_id)) from leapflow.hardware.node_gate import control_safety_refusal from leapflow.hardware.lhp.protocol import FAILURE_SAFETY_DENIED @@ -595,7 +637,7 @@ async def open( ) # Re-check after the awaited precheck: another open may have won the device. if self.busy_device((device_id,)): - return _error(FAILURE_DEVICE_BUSY, f"device {device_id!r} is held by another session") + return _error(FAILURE_DEVICE_BUSY, self.busy_message(device_id)) if self._registry.context(device_id) is not context: return _error(FAILURE_SAFETY_DENIED, "device declaration changed during control precheck") if reason := control_safety_refusal(self._registry, device_id): diff --git a/src/leapflow/hardware/lhp/node_lifecycle.py b/src/leapflow/hardware/lhp/node_lifecycle.py new file mode 100644 index 00000000..9b53185c --- /dev/null +++ b/src/leapflow/hardware/lhp/node_lifecycle.py @@ -0,0 +1,613 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""Find and stop the LHP node served on this machine -- ``leap hw stop``. + +``leap hw serve`` is a foreground process that shuts down gracefully on +SIGINT/SIGTERM (``runtime.py``). This module is its counterpart: it locates the +serving process, asks whoever owns it to stop it, and waits for the exit, so +the caller learns whether the graceful shutdown actually ran. + +Discovery has two sources: + +- **Node records.** ``serve_node`` writes ``/lhp-node--.json`` + (pid, process start identity, endpoint) once it is listening, and records + cleanup completion on exit. Stale records are deleted during discovery. +- **Process scan** (Linux ``/proc`` only). Legacy nodes without records are + matched by executable, command line, user, profile, and runtime directory. + +Ownership decides the stop path. A node running inside a systemd ``.service`` +unit is stopped by systemd: signalling the child directly would leave the +supervisor to judge the exit, and a unit with ``Restart=always`` would start a +rival owner of the CAN bus and cameras. An unsupervised node receives SIGTERM, +after revalidating its command line and creation identity, using a pidfd +where available. No broad process-name kill or automatic SIGKILL is used. +""" + +from __future__ import annotations + +import json +import logging +import math +import os +import re +import shlex +import signal +import subprocess +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterator, Sequence + +from leapflow.hardware.lhp.protocol import LHP_DEFAULT_PORT, is_valid_device_id + +logger = logging.getLogger(__name__) + +NODE_ID_ENV = "LEAPFLOW_LHP_NODE_ID" +RECORD_PREFIX = "lhp-node-" +RECORD_VERSION = 1 + +SOURCE_RECORD = "record" +SOURCE_SCAN = "process_scan" + +STATUS_STOPPED = "stopped" +STATUS_NOT_RUNNING = "not_running" +STATUS_TIMEOUT = "timeout" +STATUS_REFUSED = "refused" + +SCOPE_USER = "user" +SCOPE_SYSTEM = "system" + +# Above the serve shutdown budget (per-actuator halt 2 s, skill unwind grace, +# planning-worker join) and the shipped unit's TimeoutStopSec=20. +DEFAULT_STOP_TIMEOUT_S = 30.0 +_SUBPROCESS_TIMEOUT_S = 10.0 +_POLL_S = 0.2 + +_UNIT_RE = re.compile(r"[^/]+\.service") + + +@dataclass(frozen=True) +class NodeProcess: + """One serving node process, identified by pid *and* creation identity.""" + + pid: int + node_id: str + start: str + endpoint: str = "" + argv: tuple[str, ...] = () + source: str = SOURCE_RECORD + record_path: Path | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "pid": self.pid, + "node_id": self.node_id, + "endpoint": self.endpoint, + "source": self.source, + "record_path": str(self.record_path) if self.record_path else "", + } + + +@dataclass(frozen=True) +class Supervisor: + """The systemd unit that owns a node process.""" + + scope: str + unit: str + + +@dataclass(frozen=True) +class StopResult: + """Outcome of one stop attempt; ``consequences`` describe the hardware after it.""" + + ok: bool + status: str + node: NodeProcess | None = None + via: str = "" + forced: bool = False + detail: str = "" + survivors: tuple[int, ...] = () + consequences: tuple[str, ...] = field(default_factory=tuple) + cleanup_complete: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "status": self.status, + "node": self.node.to_dict() if self.node is not None else None, + "via": self.via, + "forced": self.forced, + "detail": self.detail, + "survivors": list(self.survivors), + "consequences": list(self.consequences), + "cleanup_complete": self.cleanup_complete, + "hardware_state_verified": False, + } + + +# ── Process identity ───────────────────────────────────────────────────── + + +def process_start(pid: int) -> str | None: + """Creation identity of a live process, or None when it is gone (or a zombie). + + A pid alone is not a safe target: it is recycled. Linux reads the start + tick from ``/proc``; elsewhere ``ps`` reports the start time. + """ + if pid <= 1: + return None + stat_path = Path(f"/proc/{pid}/stat") + if Path("/proc/self/stat").exists(): + try: + fields = stat_path.read_text().rsplit(")", 1)[1].split() + return None if fields[0] == "Z" else fields[19] + except FileNotFoundError: + return None + except (OSError, IndexError) as exc: + raise OSError(f"Cannot inspect process {pid}; no stop confirmation") from exc + try: + completed = subprocess.run( + ["ps", "-o", "stat=,lstart=", "-p", str(pid)], + capture_output=True, text=True, timeout=_SUBPROCESS_TIMEOUT_S, check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise OSError("Cannot inspect process identity with ps") from exc + values = completed.stdout.strip().split(maxsplit=1) + if not values or values[0].startswith("Z"): + return None + if completed.returncode != 0 or len(values) != 2: + raise OSError("Process identity query failed") + return values[1] + + +def is_alive(node: NodeProcess) -> bool: + return process_start(node.pid) == node.start + + +def child_processes(pid: int) -> tuple[tuple[int, str], ...]: + """Snapshot descendants, including workers spawned by non-main threads.""" + result = subprocess.run( + ["ps", "-axo", "pid=,ppid="], capture_output=True, text=True, + timeout=_SUBPROCESS_TIMEOUT_S, check=False, + ) + if result.returncode != 0: + raise OSError("Cannot inspect the node's worker processes") + parents: dict[int, list[int]] = {} + for line in result.stdout.splitlines(): + values = line.split() + if len(values) == 2 and all(value.isdigit() for value in values): + child, parent = map(int, values) + parents.setdefault(parent, []).append(child) + children: dict[int, str] = {} + pending, seen = [pid], {pid} + while pending: + for child in parents.get(pending.pop(), ()): + if child in seen: + continue + seen.add(child) + start = process_start(child) + if start is not None: + children[child] = start + pending.append(child) + return tuple(children.items()) + + +def supervisor_of(pid: int) -> Supervisor | None: + """The systemd ``.service`` unit whose cgroup holds *pid*, if any. + + A terminal-started node lives in a session ``.scope`` and has no + supervisor; ``user@.service`` is the user manager itself, not a unit + that owns the node. + """ + try: + lines = Path(f"/proc/{pid}/cgroup").read_text().splitlines() + except FileNotFoundError: + if Path("/proc/self/cgroup").exists() and process_start(pid) is not None: + raise OSError("Cannot determine the node's supervisor; no signal sent") + return None + for line in lines: + path = line.split(":", 2)[-1] + for component in reversed(path.split("/")): + if _UNIT_RE.fullmatch(component) and not component.startswith("user@"): + scope = SCOPE_USER if "/user@" in path else SCOPE_SYSTEM + return Supervisor(scope=scope, unit=component) + return None + + +# ── Node records (written by serve_node) ───────────────────────────────── + + +def _replace_record(path: Path, payload: dict[str, Any]) -> None: + """Publish a private receipt atomically, without following a preexisting temp symlink.""" + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, prefix=path.stem + "-", + suffix=".tmp", delete=False, + ) as stream: + temporary = Path(stream.name) + json.dump(payload, stream) + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def write_record(runtime_dir: Path, *, node_id: str, endpoint: str) -> Path | None: + """Record this process as the server of *node_id* without storing credentials.""" + if not is_valid_device_id(node_id): + raise ValueError("Invalid node id for lifecycle record") + pid = os.getpid() + start = process_start(pid) + if start is None: + logger.warning("Node record skipped: process start identity unavailable") + return None + path = Path(runtime_dir) / f"{RECORD_PREFIX}{node_id}-{pid}.json" + payload = { + "version": RECORD_VERSION, + "node_id": node_id, + "pid": pid, + "start": start, + "endpoint": endpoint, + "started_at": time.time(), + "cleanup_complete": False, + } + try: + path.parent.mkdir(parents=True, exist_ok=True) + _replace_record(path, payload) + except OSError: + logger.warning("Node record %s could not be written; `leap hw stop` falls back to a process scan", + path, exc_info=True) + return None + return path + + +def finish_record(path: Path | None, *, complete: bool) -> None: + """Leave a cleanup receipt; it does not certify any hardware's physical state.""" + if path is None: + return + try: + data = json.loads(path.read_text()) + data["cleanup_complete"] = complete + _replace_record(path, data) + except (OSError, ValueError, TypeError): + logger.warning("Cannot record node cleanup outcome", exc_info=True) + + +def remove_record(path: Path | None) -> None: + """Remove a stale node record, leaving live records untouched by discovery.""" + if path is None: + return + try: + path.unlink(missing_ok=True) + except OSError: + logger.warning("Node record %s could not be removed", path, exc_info=True) + + +def _recorded(runtime_dir: Path) -> Iterator[NodeProcess]: + for path in sorted(Path(runtime_dir).glob(f"{RECORD_PREFIX}*.json")): + try: + if path.is_symlink() or path.stat().st_uid != os.getuid(): + continue + data = json.loads(path.read_text()) + if not isinstance(data, dict) or data.get("version") != RECORD_VERSION: + continue + if type(data.get("pid")) is not int or data["pid"] <= 1: + continue + if not is_valid_device_id(str(data.get("node_id", ""))): + continue + node = NodeProcess( + pid=int(data["pid"]), node_id=str(data["node_id"]), start=str(data["start"]), + endpoint=str(data.get("endpoint") or ""), source=SOURCE_RECORD, record_path=path, + ) + except (OSError, ValueError, KeyError, TypeError): + logger.warning("Ignoring unreadable node record %s", path) + continue + if not is_alive(node): + remove_record(path) + elif _valid_target(node): + yield node + + +# ── Process scan (Linux) ───────────────────────────────────────────────── + + +def is_serve_argv(argv: Sequence[str]) -> bool: + """True for ``leap hw serve ...`` and ``python -m leapflow.hardware.lhp ...``.""" + if not argv: + return False + parts = list(argv) + if re.fullmatch(r"python(?:\d+(?:\.\d+)*)?", Path(parts[0]).name): + parts.pop(0) + while parts and parts[0] in ("-I", "-u", "-B", "-s", "-S", "-E"): + parts.pop(0) + if parts[:2] == ["-m", "leapflow.hardware.lhp"]: + return "--help" not in parts and "-h" not in parts + if parts[:2] == ["-m", "leapflow"]: + parts = parts[2:] + elif parts and Path(parts[0]).name == "leap": + parts = parts[1:] + else: + return False + elif Path(parts[0]).name == "leap": + parts = parts[1:] + else: + return False + if parts[:1] == ["--mock-host"]: + parts.pop(0) + return parts[:2] == ["hw", "serve"] and "--help" not in parts and "-h" not in parts + + +def _flag(argv: Sequence[str], name: str) -> str: + for index, part in enumerate(argv): + if part == name and index + 1 < len(argv): + return argv[index + 1] + if part.startswith(name + "="): + return part.split("=", 1)[1] + return "" + + +def _endpoint_from_argv(argv: Sequence[str]) -> str: + unix_path = _flag(argv, "--unix") or _flag(argv, "--unix-path") + if unix_path: + return f"unix://{unix_path}" + host = _flag(argv, "--host") or "127.0.0.1" + return f"tcp://{host}:{_flag(argv, '--port') or LHP_DEFAULT_PORT}" + + +def _process_environment(pid: int) -> dict[str, str]: + # Only identity/path fields; credentials are never retained or printed. + names = {NODE_ID_ENV, "LEAPFLOW_PROFILE", "LEAPFLOW_DATA_DIR", "LEAPFLOW_RUNTIME_DIR"} + entries = Path(f"/proc/{pid}/environ").read_bytes().split(b"\0") + result = {} + for entry in entries: + name, _, value = entry.partition(b"=") + key = name.decode(errors="replace") + if key in names: + result[key] = value.decode(errors="replace").strip() + return result + + +def _process_argv(pid: int) -> tuple[str, ...]: + if Path("/proc/self/stat").exists(): + raw = Path(f"/proc/{pid}/cmdline").read_bytes() + if Path(f"/proc/{pid}").stat().st_uid != os.getuid(): + return () + return tuple(os.fsdecode(part) for part in raw.split(b"\0") if part) + result = subprocess.run( + ["ps", "-ww", "-o", "uid=,command=", "-p", str(pid)], capture_output=True, + text=True, timeout=_SUBPROCESS_TIMEOUT_S, check=False, + ) + fields = result.stdout.strip().split(maxsplit=1) + if result.returncode or len(fields) != 2 or fields[0] != str(os.getuid()): + return () + return tuple(shlex.split(fields[1])) + + +def _valid_target(node: NodeProcess) -> bool: + if node.pid == os.getpid() or not is_alive(node): + return False + argv = _process_argv(node.pid) + if not is_serve_argv(argv): + return False + return not node.argv or node.argv == argv + + +def _runtime_from_environment(env: dict[str, str]) -> Path: + root = Path(env.get("LEAPFLOW_DATA_DIR") or "~/.leapflow").expanduser() + profile = env.get("LEAPFLOW_PROFILE") or "default" + from leapflow.layout import build_layout + + return Path(env.get("LEAPFLOW_RUNTIME_DIR") or build_layout(root).profile(profile).runtime_dir).expanduser() + + +def _scanned(runtime_dir: Path, proc_root: Path = Path("/proc")) -> Iterator[NodeProcess]: + if not (proc_root / "self").exists(): + return + uid, me = os.getuid(), os.getpid() + for entry in proc_root.iterdir(): + if not entry.name.isdigit() or int(entry.name) == me: + continue + try: + if entry.stat().st_uid != uid: + continue + raw = (entry / "cmdline").read_bytes() + except OSError: + continue + argv = tuple(part.decode(errors="replace") for part in raw.split(b"\0") if part) + if not is_serve_argv(argv): + continue + pid = int(entry.name) + start = process_start(pid) + if start is None: + continue + try: + env = _process_environment(pid) + except OSError: + continue # Unknown profile is not authority to stop a process. + if _runtime_from_environment(env).resolve() != Path(runtime_dir).resolve(): + continue + yield NodeProcess( + pid=pid, + node_id=_flag(argv, "--node-id") or env.get(NODE_ID_ENV, ""), + start=start, + endpoint=_endpoint_from_argv(argv), + argv=argv, + source=SOURCE_SCAN, + ) + + +def discover_nodes(runtime_dir: Path, node_id: str = "") -> list[NodeProcess]: + """Live nodes on this machine, optionally only *node_id*; records win over the scan.""" + nodes = {node.pid: node for node in _recorded(runtime_dir)} + for node in _scanned(runtime_dir): + nodes.setdefault(node.pid, node) + return sorted( + (node for node in nodes.values() if not node_id or node.node_id == node_id), + key=lambda node: node.pid, + ) + + +# ── Stop ───────────────────────────────────────────────────────────────── + + +def _send(node: NodeProcess, signum: int) -> None: + """Signal *node* only while it is still the same process; raises ProcessLookupError if not.""" + if hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"): + descriptor = os.pidfd_open(node.pid, 0) + try: + if not _valid_target(node): + raise ProcessLookupError(node.pid) + signal.pidfd_send_signal(descriptor, signum) + finally: + os.close(descriptor) + return + if not _valid_target(node): + raise ProcessLookupError(node.pid) + os.kill(node.pid, signum) + + +def _systemctl(*args: str) -> str: + result = subprocess.run( + ["systemctl", "--user", *args], capture_output=True, text=True, + timeout=_SUBPROCESS_TIMEOUT_S, check=False, + ) + if result.returncode: + raise OSError("systemctl failed; inspect the owning unit and user manager") + return result.stdout.strip() + + +def _unit_state(owner: Supervisor) -> dict[str, str]: + output = _systemctl("show", "--property=MainPID,ActiveState,Result", "--", owner.unit) + return dict(line.split("=", 1) for line in output.splitlines() if "=" in line) + + +def _cleanup_receipt(node: NodeProcess) -> bool | None: + if node.record_path is None: + return None + try: + data = json.loads(node.record_path.read_text()) + if data.get("pid") == node.pid and data.get("start") == node.start: + result = data.get("cleanup_complete") + return result if type(result) is bool else None + except (OSError, ValueError, AttributeError): + pass + return None + + +def _wait_stopped( + node: NodeProcess, children: tuple[tuple[int, str], ...], owner: Supervisor | None, + deadline: float, +) -> tuple[bool, tuple[int, ...], dict[str, str]]: + tracked = dict(children) + while True: + alive = is_alive(node) + if alive: + tracked.update(child_processes(node.pid)) + survivors = tuple(pid for pid, start in tracked.items() if process_start(pid) == start) + state = _unit_state(owner) if owner is not None else {} + unit_stopped = owner is None or ( + state.get("MainPID") == "0" and state.get("ActiveState") in ("inactive", "failed") + ) + if not alive and not survivors and unit_stopped: + return True, (), state + if time.monotonic() >= deadline: + return False, survivors, state + time.sleep(min(_POLL_S, max(0.0, deadline - time.monotonic()))) + + +def stop_node(node: NodeProcess, *, timeout_s: float = DEFAULT_STOP_TIMEOUT_S) -> StopResult: + """Request graceful shutdown, then verify owner and worker exit; never force-kill.""" + if not math.isfinite(timeout_s) or timeout_s <= 0: + return StopResult(ok=False, status=STATUS_REFUSED, detail="--timeout must be positive and finite") + via = "" + deadline = time.monotonic() + timeout_s + try: + if not is_alive(node): + return StopResult(ok=True, status=STATUS_NOT_RUNNING, node=node, + detail="The node had already exited; no signal sent.") + if not _valid_target(node): + return StopResult(ok=False, status=STATUS_REFUSED, node=node, + detail="Process identity no longer matches hw serve; no signal sent.") + owner = supervisor_of(node.pid) + unit = owner.unit if owner is not None else "" + if owner is not None and owner.scope != SCOPE_USER: + return StopResult(ok=False, status=STATUS_REFUSED, node=node, + detail=f"System unit {unit} owns this node; ask its administrator to stop it.") + children = child_processes(node.pid) + if owner is not None: + state = _unit_state(owner) + if state.get("MainPID") != str(node.pid) or not _valid_target(node): + return StopResult(ok=False, status=STATUS_REFUSED, node=node, + detail="The node is not the unit's main process; refusing to stop unrelated services.") + via = f"systemd:{unit}" + _systemctl("--no-block", "stop", "--", unit) + else: + via = "SIGTERM" + try: + _send(node, signal.SIGTERM) + except ProcessLookupError: + if is_alive(node): + raise + # Parent exit raced delivery: still check its captured workers. + logger.info("Requested LHP node stop: node=%s pid=%d via=%s", node.node_id, node.pid, via) + exited, survivors, state = _wait_stopped(node, children, owner, deadline) + forced = state.get("Result") in ("timeout", "signal", "core-dump") + if not exited: + return StopResult( + ok=False, status=STATUS_TIMEOUT, node=node, via=via, forced=forced, + detail="Node, workers, or supervisor still active. Check the node log and use a longer --timeout; no SIGKILL sent.", + survivors=survivors, consequences=("Shutdown is incomplete; hardware state is unknown.",), + ) + receipt = _cleanup_receipt(node) + complete = receipt is True + unclean = receipt is False or state.get("Result", "success") != "success" + if node.record_path is not None: + remove_record(node.record_path) + return StopResult( + ok=not unclean, status="stopped_uncleanly" if unclean else STATUS_STOPPED, node=node, + via=via, forced=forced, cleanup_complete=complete, + detail="Node and observed workers exited." if not unclean else "Node cleanup or supervisor reported an unclean stop; inspect hardware and logs.", + consequences=stop_consequences(forced=forced, supervisor=unit, complete=complete), + ) + except (OSError, subprocess.SubprocessError, ValueError) as exc: + logger.error("LHP node stop could not be confirmed", exc_info=True) + return StopResult(ok=False, status=STATUS_REFUSED, node=node, via=via, + detail=f"Stop refused or unconfirmed: {type(exc).__name__}. Inspect the node log and supervisor.") + + +def stop_consequences(*, forced: bool, supervisor: str = "", complete: bool = False) -> tuple[str, ...]: + """Expected shutdown effects, explicitly separate from physical verification.""" + lines = [ + "Graceful cleanup completed." if complete and not forced else + "Graceful cleanup is unconfirmed; inspect the node log. A process exit does not prove a safe hardware stop.", + "Shutdown requests cancellation of runs and leases, attempts declared actuator halts, then releases camera previews/streams and device links.", + "Piper: halt attempts a powered hold-in-place; close only disconnects CAN. No automatic disable, reset, parking move, or gripper-open command is sent.", + "Service stop is NOT power-off. An enabled arm may remain energized and the gripper may retain its last command; posture, torque, and object retention after disconnect are NOT verified. Other devices follow their driver/firmware semantics.", + "Node-side monitoring and lease deadman are no longer running; host remote control/estop through this node is unavailable. Keep hardware attended and use the device's physical safety procedures if necessary.", + "LHP connections close; host fleet marks devices unreachable/offline according to its configured heartbeat. The host daemon/TUI and unrelated services are not stopped.", + ] + if forced: + lines.insert(0, "WARNING: supervisor forced or abnormal termination; cleanup and hardware halt cannot be assumed. Motion may continue.") + if supervisor: + lines.append(f"User unit {supervisor} was stopped explicitly, preventing its restart policy from restarting this run. This does not disable boot activation; resume with `systemctl --user start {supervisor}`.") + else: + lines.append("Resume with the same `leap hw serve ...` command and profile; an external supervisor, if any, may have its own restart policy.") + return tuple(lines) + + +__all__ = [ + "DEFAULT_STOP_TIMEOUT_S", + "NodeProcess", + "StopResult", + "Supervisor", + "discover_nodes", + "is_serve_argv", + "process_start", + "remove_record", + "finish_record", + "stop_consequences", + "stop_node", + "supervisor_of", + "write_record", +] diff --git a/src/leapflow/hardware/lhp/runtime.py b/src/leapflow/hardware/lhp/runtime.py index 9fcb9cf8..0cccc651 100644 --- a/src/leapflow/hardware/lhp/runtime.py +++ b/src/leapflow/hardware/lhp/runtime.py @@ -16,7 +16,8 @@ Stopping is graceful on SIGINT/SIGTERM: the listener closes, live connections are dropped, and node-side transports are released before the -process exits. +process exits. ``leap hw stop`` (``node_lifecycle.py``) delivers that stop +through the process's owner, using the node record written once listening. """ from __future__ import annotations @@ -28,10 +29,12 @@ import os import signal import sys +from contextlib import AsyncExitStack from pathlib import Path from typing import Any, Sequence from leapflow.hardware.lhp.authority import WRITE_MODE_PERMIT_REQUIRED +from leapflow.hardware.lhp.node_lifecycle import NODE_ID_ENV, finish_record, write_record from leapflow.hardware.lhp.protocol import ( LHP_DEFAULT_PORT, LHP_PROTOCOL_VERSION, @@ -41,7 +44,7 @@ logger = logging.getLogger(__name__) -_NODE_ID_ENV = "LEAPFLOW_LHP_NODE_ID" +_NODE_ID_ENV = NODE_ID_ENV _TOKEN_ENV = "LEAPFLOW_LHP_TOKEN" _WRITE_MODE_ENV = "LEAPFLOW_LHP_WRITE_MODE" _PERMIT_KEYS_ENV = "LEAPFLOW_LHP_PERMIT_ISSUER_KEYS" @@ -172,12 +175,17 @@ async def serve_node( loop.add_signal_handler(sig, stop.set) except NotImplementedError: # pragma: no cover - platform without loop signals pass + record = None try: if unix_path: endpoint = await server.start_unix(unix_path) else: bound_host, bound_port = await server.start(host=host, port=port) endpoint = f"tcp://{bound_host}:{bound_port}" + # Lets `leap hw stop` find this process without a pid guess. + record = await asyncio.to_thread( + write_record, config.runtime_dir, node_id=node_id, endpoint=endpoint, + ) ready = { "ok": True, "event": "lhp_node_ready", @@ -198,11 +206,19 @@ async def serve_node( logger.info("LHP node %r serving %d device(s) at %s", node_id, len(contexts), endpoint) await stop.wait() finally: - await server.stop() - await skill_runtime.close() - if worker is not None: - await worker.close() - await registry.close_all() + complete = False + try: + async with AsyncExitStack() as cleanup: + # LIFO: stop admission/leases, then skills, compute, and transports. + # One failing closer must not skip the remaining owners' cleanup. + cleanup.push_async_callback(registry.close_all) + if worker is not None: + cleanup.push_async_callback(worker.close) + cleanup.push_async_callback(skill_runtime.close) + cleanup.push_async_callback(server.stop) + complete = True + finally: + await asyncio.to_thread(finish_record, record, complete=complete) logger.info("LHP node %r stopped", node_id) return 0 diff --git a/src/leapflow/hardware/lhp/server.py b/src/leapflow/hardware/lhp/server.py index cccf80a7..adf20d6b 100644 --- a/src/leapflow/hardware/lhp/server.py +++ b/src/leapflow/hardware/lhp/server.py @@ -692,7 +692,7 @@ def _write_feasibility_refusal(self, device_id: str) -> WriteOutcome | None: """ if self._sessions.busy_device((device_id,)): return refusal( - f"device {device_id!r} is held by a running session; stop it or wait for it to end", + self._sessions.busy_message(device_id), FAILURE_DEVICE_BUSY, ) from leapflow.hardware.node_gate import recovery_safety_refusal diff --git a/src/leapflow/hardware/lhp/skill_methods.py b/src/leapflow/hardware/lhp/skill_methods.py index 50f2f60f..90294931 100644 --- a/src/leapflow/hardware/lhp/skill_methods.py +++ b/src/leapflow/hardware/lhp/skill_methods.py @@ -142,8 +142,8 @@ async def reload(self, params: Mapping[str, Any]) -> dict[str, Any]: node_session_id=require_str(params, "node_session_id")) if stale is not None: return _outcome(stale) - if self._sessions.busy_device(prepared.devices): - return _outcome(SkillOutcome.failed(FAILURE_DEVICE_BUSY, "reload devices are held by a session")) + if busy := self._sessions.busy_device(prepared.devices): + return _outcome(SkillOutcome.failed(FAILURE_DEVICE_BUSY, self._sessions.busy_message(busy))) if self._authority.mode != WRITE_MODE_PERMIT_REQUIRED: return _outcome(SkillOutcome.failed(FAILURE_WRITE_REFUSED, "live reload requires permit_required authority")) refusal = self._authority.check("skill.reload", params) @@ -209,7 +209,7 @@ async def execute(self, params: Mapping[str, Any]) -> dict[str, Any]: busy = self._sessions.busy_device(runtime.plan_devices(plan)) if busy: return _outcome( - SkillOutcome.failed(FAILURE_DEVICE_BUSY, f"device {busy!r} is held by a session") + SkillOutcome.failed(FAILURE_DEVICE_BUSY, self._sessions.busy_message(busy)) ) refusal, admission = self._authority.admit_run( params, expected_digest=runtime.permit_digest(plan) diff --git a/src/leapflow/hardware/node_gate.py b/src/leapflow/hardware/node_gate.py index b20eda22..dc6f8b31 100644 --- a/src/leapflow/hardware/node_gate.py +++ b/src/leapflow/hardware/node_gate.py @@ -238,6 +238,7 @@ async def write(self, device_id: str, channel_id: str, value: Any) -> WriteOutco token = None attempted = [False] settled = False + detail = "dispatch ended without a settlement verdict" if sessions is not None: try: token = sessions.begin_dispatch(device_id) @@ -246,10 +247,11 @@ async def write(self, device_id: str, channel_id: str, value: Any) -> WriteOutco try: outcome = await self._write(device_id, channel_id, value, attempted) settled = outcome.side_effect_state != SIDE_EFFECT_UNKNOWN + detail = outcome.error or outcome.failure_code or detail return outcome finally: if token is not None: - sessions.finish_dispatch(token, settled=settled or not attempted[0]) + sessions.finish_dispatch(token, settled=settled or not attempted[0], detail=detail) async def _write(self, device_id: str, channel_id: str, value: Any, attempted: list[bool]) -> WriteOutcome: context, channel = self._resolve(device_id, channel_id) @@ -280,6 +282,7 @@ async def write_batch( token = None attempted = [False] settled = False + detail = "dispatch ended without a settlement verdict" if sessions is not None: try: token = sessions.begin_dispatch(device_id) @@ -289,10 +292,12 @@ async def write_batch( try: ok, outcomes = await self._write_batch(device_id, commands, attempted) settled = all(o.side_effect_state != SIDE_EFFECT_UNKNOWN for o in outcomes) + detail = next((o.error or o.failure_code or detail for o in outcomes + if o.side_effect_state == SIDE_EFFECT_UNKNOWN), detail) return ok, outcomes finally: if token is not None: - sessions.finish_dispatch(token, settled=settled or not attempted[0]) + sessions.finish_dispatch(token, settled=settled or not attempted[0], detail=detail) async def _write_batch( self, device_id: str, commands: Sequence[tuple[str, Any]], attempted: list[bool] diff --git a/src/leapflow/hardware/observability/digest.py b/src/leapflow/hardware/observability/digest.py index 64524f24..b35e901a 100644 --- a/src/leapflow/hardware/observability/digest.py +++ b/src/leapflow/hardware/observability/digest.py @@ -88,6 +88,7 @@ def build_digest(registry: Any, *, now: float | None = None) -> HardwareDigest: outcomes=tuple(outcomes), calibration=_calibration(registry, contexts, now=moment), storage=_storage(registry), + conditions=tuple(_safely(getattr(registry, "observation_conditions", lambda: ()), default=())), ) @@ -232,6 +233,7 @@ def _events(registry: Any) -> tuple[dict[str, Any], ...]: "title": f"{kind} · {device_id}.{channel_id}", "summary": str(getattr(event, "detail", "")), "severity": _event_severity(kind), + "observation": dict(getattr(event, "observation", {}) or {}), "kind": kind, "device_id": device_id, "channel_id": channel_id, @@ -264,6 +266,7 @@ def _events(registry: Any) -> tuple[dict[str, Any], ...]: "sample_loss", "calibration_failed", "calibration_expired", + "observation_warning", # A degradation policy fired while actions are disabled: worth a warning, # but nothing was stopped, so it must not page anyone like an alert. "degradation_suppressed", diff --git a/src/leapflow/hardware/observability/producer.py b/src/leapflow/hardware/observability/producer.py index 900a0cc4..1c206ddb 100644 --- a/src/leapflow/hardware/observability/producer.py +++ b/src/leapflow/hardware/observability/producer.py @@ -1,5 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -"""The ``hardware`` monitor domain: one finding per cycle, carrying the digest. +"""The ``hardware`` monitor domain: episode notifications and current-state digests. The only file here with a side effect, and the only one the daemon wires. It implements ``MonitorProducer`` so device observation reaches the board through the @@ -14,11 +14,15 @@ from __future__ import annotations +import asyncio +import hashlib import logging from typing import Any, Callable, Sequence -from leapflow.hardware.advisory import active_advisories +from leapflow.hardware.advisory import ADVISORY_KINDS from leapflow.hardware.observability.digest import ALERT_KINDS, build_digest +from leapflow.hardware.observation import Episode +from leapflow.hardware.observability.series import _fit from leapflow.monitor.types import Evidence, Finding, ProducerContext, Severity logger = logging.getLogger(__name__) @@ -33,11 +37,9 @@ alert and the producer still declined to push it to anybody. """ -_DEGRADED_QUALITIES = frozenset({"stale", "saturated"}) - class HardwareObservationProducer: - """Observe every admitted device and emit one finding describing the bench. + """Observe every admitted condition, independently of chart and event-ring limits. Takes a provider rather than the registry itself, because the registry's persistence and experience store are bound during deferred initialization -- @@ -46,15 +48,20 @@ class HardwareObservationProducer: it captured before it was fully wired. """ - def __init__(self, registry_provider: Callable[[], Any]) -> None: + def __init__( + self, registry_provider: Callable[[], Any], *, finding_store: Any = None, run_db: Any = None + ) -> None: self._registry_provider = registry_provider + self._finding_store = finding_store + self._run_db = run_db + self._checkpoints: dict[str, dict[str, Any]] = {} @property def domain(self) -> str: return DOMAIN async def observe(self, ctx: ProducerContext) -> Sequence[Finding]: - """Return at most one finding. Never raises.""" + """Return uncheckpointed transitions or a current-state digest.""" try: registry = self._registry_provider() except Exception as exc: # noqa: BLE001 - a board panel is not worth a failed watch @@ -70,35 +77,135 @@ async def observe(self, ctx: ProducerContext) -> Sequence[Finding]: # store with rows saying nothing happened. return [] - severity, headline = _assess(digest) + if self._finding_store is not None: + + def query() -> dict[str, dict[str, Any]]: + return self._finding_store.latest_notification_checkpoints(DOMAIN) + + try: + checkpoints = ( + await self._run_db(query) if self._run_db else await asyncio.to_thread(query) + ) + except Exception: # noqa: BLE001 - failed checkpoint reads must not re-page incidents + logger.warning("hardware watch: checkpoint read failed", exc_info=True) + return [] + else: + checkpoints = dict(self._checkpoints) + acknowledge = getattr(registry, "acknowledge_observation", None) + if callable(acknowledge): + acknowledge(checkpoints) + findings = [] + monitored = {row["key"] for row in digest.conditions} + conditions = list(digest.conditions) + # Command-path faults and older/external producers may not have a streaming + # detector. Their individual evidence is still reported once, never as a set + # of whatever happens to remain in the event ring. + for event in reversed(digest.events): + kind = str(event.get("kind", "")) + condition = f"advisory:{event.get('quantity', '')}" if kind in ADVISORY_KINDS else kind + key = f"{event.get('device_id', '')}/{event.get('channel_id', '')}/{condition}" + if (kind not in _ALERT_EVENTS and kind not in ADVISORY_KINDS) or key in monitored: + continue + episode = hashlib.sha256( + repr((key, event.get("x"), event.get("detail"))).encode() + ).hexdigest()[:24] + phase = "recovery" if kind == "advisory_cleared" else "entry" + conditions.append( + { + "key": key, + "pending": ( + { + "key": key, + "condition": condition, + "episode": episode, + "runtime_id": episode, + "revision": 1, + "phase": phase, + "active": phase != "recovery", + "known": True, + "reason": event.get("detail", ""), + "device_id": event.get("device_id", ""), + "channel_id": event.get("channel_id", ""), + "observed_at": event.get("x", 0.0), + "event_only": True, + "identity": episode, + }, + ), + } + ) + base = digest.to_payload() + for row in conditions: + for notice in row.get("pending", ()): + checkpoint = checkpoints.get(row["key"], {}) + notification = _notification(notice, checkpoint, ctx.now) + if notification is None: + continue + checkpoints[row["key"]] = notification + phase, condition = notification["phase"], notice["condition"] + hard = condition in _ALERT_EVENTS + severity = Severity.NOTABLE if phase == "recovery" or not hard else Severity.ALERT + where = f"{notice.get('device_id', '')}.{notice.get('channel_id', '')}" + detail = ( + f"{where}: {notice.get('reason', condition)}; raw={notice.get('value')}, " + f"soft={notice.get('soft_min')}..{notice.get('soft_max')}, " + f"hard={notice.get('hard_min')}..{notice.get('hard_max')}, " + f"margin={notice.get('margin', 0):g}, duration={notice.get('duration_s', 0):.2f}s" + ) + payload = _fit({**base, "notification": notification}) + findings.append( + Finding( + watch_id=ctx.spec.watch_id or DOMAIN, + domain=DOMAIN, + title=f"Hardware {condition} {phase}: {where}", + summary=detail, + severity=severity, + ts=ctx.now, + dedup_key=f"hardware:{row['key']}:{notification['episode']}:{notification['sequence']}", + tags=(DOMAIN, condition, phase), + evidence=(Evidence(kind="metric", label=where, value=detail),), + payload=payload, + ) + ) + if self._finding_store is None: + self._checkpoints = checkpoints + if findings: + return findings + _, headline = _assess(digest) + # Current-state snapshots feed the existing board. Historical events are + # evidence only; an unchanged condition cannot notify through chart churn. + snapshot_severity = Severity.INFO + if int(digest.storage.get("write_failures", 0) or 0) > 0: + snapshot_severity, headline = Severity.NOTABLE, "Sampled history is not being persisted" + elif any(_is_behind(row) for row in digest.sampling): + snapshot_severity, headline = Severity.NOTABLE, "Sampling behind declared rate" return [ Finding( watch_id=ctx.spec.watch_id or DOMAIN, domain=DOMAIN, title=headline, summary=_summary(digest), - severity=severity, - ts=digest.generated_at, - dedup_key=f"hardware:{_state_fingerprint(digest)}", - tags=("hardware", *sorted({str(event["kind"]) for event in digest.events})), + severity=snapshot_severity, + ts=ctx.now, + dedup_key=f"hardware:snapshot:{_state_fingerprint(digest)}", + tags=(DOMAIN,), evidence=_evidence(digest), - payload=digest.to_payload(), + payload=base, ) ] def _assess(digest: Any) -> tuple[Severity, str]: """Return the severity and one-line headline for this cycle.""" - kinds = {str(event["kind"]) for event in digest.events} - breaching = sorted(kinds & _ALERT_EVENTS) + breaching = sorted( + { + str(row.get("condition", "")) + for row in digest.conditions + if row.get("active") and row.get("known") and row.get("condition") in _ALERT_EVENTS + } + ) if breaching: return Severity.ALERT, f"Device envelope event: {', '.join(breaching)}" - degraded = [s for s in digest.series if s.quality_worst in _DEGRADED_QUALITIES] - if degraded: - names = ", ".join(sorted(s.id for s in degraded)[:3]) - return Severity.NOTABLE, f"Channel quality degraded: {names}" - if int(digest.storage.get("write_failures", 0) or 0) > 0: # Not cosmetic: dropped windows are the one storage fault that leaves no # trace in the data, so the count is the only way it is ever noticed. @@ -109,11 +216,11 @@ def _assess(digest: Any) -> tuple[Severity, str]: names = ", ".join(sorted(str(row.get("channel_id", "")) for row in shortfall)[:3]) return Severity.NOTABLE, f"Sampling behind declared rate: {names}" - advisories = active_advisories(digest.events) - if advisories: - names = ", ".join(sorted(advisories)[:3]) - return Severity.NOTABLE, f"Device advisory (firmware-protected, reported only): {names}" - + warnings = [row for row in digest.conditions if row.get("active") and row.get("known")] + if warnings: + return Severity.NOTABLE, f"Operational hardware warning: {warnings[0].get('key', '')}" + if not digest.conditions or any(not row.get("known") for row in digest.conditions): + return Severity.INFO, "Hardware observation unknown or awaiting fresh evidence" return Severity.INFO, f"{len(digest.devices)} device(s) within declared envelopes" @@ -163,10 +270,62 @@ def _state_fingerprint(digest: Any) -> str: from the trace itself, which changes on every cycle and would defeat dedup entirely. """ - degraded = sorted(s.id for s in digest.series if s.quality_worst != "ok") - kinds = sorted({str(event["kind"]) for event in digest.events}) + states = sorted( + (row.get("key", ""), bool(row.get("active")), bool(row.get("known"))) + for row in digest.conditions + ) failing = int(digest.storage.get("write_failures", 0) or 0) > 0 - return "|".join([",".join(degraded), ",".join(kinds), str(failing)]) + behind = sorted(str(row.get("channel_id", "")) for row in digest.sampling if _is_behind(row)) + return hashlib.sha256(repr((states, failing, behind)).encode()).hexdigest()[:24] + + +def _notification( + notice: dict[str, Any], checkpoint: dict[str, Any], now: float +) -> dict[str, Any] | None: + """Map source episodes to persisted notification progress, including restarts.""" + if ( + notice.get("event_only") + and checkpoint + and notice.get("observed_at", 0) <= checkpoint.get("observed_at", 0) + ): + return None + same_runtime = checkpoint.get("runtime_id") == notice.get("runtime_id") + if same_runtime and notice.get("revision", 0) <= checkpoint.get("revision", 0): + return None + phase = notice["phase"] + continued = ( + checkpoint.get("active") + and checkpoint.get("identity") == notice.get("identity") + and (not same_runtime or checkpoint.get("source_episode") == notice.get("episode")) + ) + if phase == "entry" and continued: + phase = "reminder" # restored progress is not restored healthy evidence + if phase == "reminder" and continued and now < checkpoint.get("next_report_at", 0.0): + return None + episode = checkpoint["episode"] if continued else notice["episode"] + sequence = ( + int(checkpoint.get("sequence", 0)) + 1 if continued else int(notice.get("sequence", 0)) + ) + after = float(notice.get("report_after_s", 60.0)) + maximum = float(notice.get("report_max_s", 900.0)) + clock = Episode(active=True, interval=float(checkpoint.get("interval", after)), next_report=0.0) + if phase == "reminder": + clock.observe( + breached=True, clear=False, now=now, report_after_s=after, report_max_s=maximum + ) + else: + clock.interval, clock.next_report = after, now + after + return { + **notice, + "phase": phase, + "episode": episode, + "sequence": sequence, + "source_episode": notice["episode"], + "active": phase != "recovery", + "next_report_at": clock.next_report, + "interval": clock.interval, + "notified_at": now, + } __all__ = ["DOMAIN", "HardwareObservationProducer"] diff --git a/src/leapflow/hardware/observability/series.py b/src/leapflow/hardware/observability/series.py index 10daafee..1d1c5645 100644 --- a/src/leapflow/hardware/observability/series.py +++ b/src/leapflow/hardware/observability/series.py @@ -165,6 +165,7 @@ class HardwareDigest: outcomes: tuple[dict[str, Any], ...] = () calibration: tuple[dict[str, Any], ...] = () storage: dict[str, Any] = field(default_factory=dict) + conditions: tuple[dict[str, Any], ...] = () @property def is_empty(self) -> bool: @@ -191,6 +192,7 @@ def to_payload(self) -> dict[str, Any]: "devices": len(self.devices), "series": len(self.series), "events": len(self.events), + "conditions": len(self.conditions), "outcomes": len(self.outcomes), "calibration": len(self.calibration), "conformance_windows": len(self.conformance), @@ -206,6 +208,8 @@ def to_payload(self) -> dict[str, Any]: "outcomes": list(self.outcomes), "calibration": list(self.calibration), "storage": dict(self.storage), + "conditions": [{key: value for key, value in row.items() if key != "pending"} + for row in self.conditions], } return _fit(payload) @@ -243,13 +247,12 @@ def decimate(points: Sequence[SeriesPoint], limit: int = MAX_POINTS) -> tuple[Se def _fit(payload: dict[str, Any]) -> dict[str, Any]: """Decimate series until the encoded payload fits ``MAX_PAYLOAD_BYTES``. - Series are the only field allowed to be large enough to need this, and they are - the only field it can reduce -- so an overrun that survives the loop means some - *other* field is unbounded, which is a producer defect this function cannot fix. - It says so at ``warning`` rather than returning quietly: the previous silent - return meant a payload four times the ceiling was persisted, pushed, and batched - into an RPC frame with no record anywhere that the ceiling had been passed. + Presentation rows may be omitted, with counts disclosed. Detector monitoring, + raw history, and the notification checkpoint remain complete and unchanged. + Copy lists because the producer shares the base digest among several findings. """ + payload = {key: list(value) if isinstance(value, list) else value + for key, value in payload.items()} limit = MAX_POINTS while limit >= 2: if _encoded_size(payload) <= MAX_PAYLOAD_BYTES: @@ -262,15 +265,20 @@ def _fit(payload: dict[str, Any]) -> dict[str, Any]: } for series in payload["series"] ] - size = _encoded_size(payload) - if size > MAX_PAYLOAD_BYTES: - logger.warning( - "hardware digest: payload is %d bytes after decimating series to the floor, " - "over the %d-byte ceiling; a non-series field is unbounded", - size, - MAX_PAYLOAD_BYTES, - ) - return payload + for key in ("conditions", "events", "outcomes", "calibration", "sampling", + "devices", "device_classes", "conformance_mix", "series"): + rows = payload.get(key, []) + while rows and _encoded_size(payload) > MAX_PAYLOAD_BYTES: + removed = max(1, len(rows) // 2) + rows = rows[:-removed] + payload[key] = rows + payload[f"{key}_omitted"] = int(payload.get(f"{key}_omitted", 0)) + removed + if _encoded_size(payload) <= MAX_PAYLOAD_BYTES: + return payload + # An irreducible oversized checkpoint is a producer defect; never persist or + # push an over-budget frame or silently alter the identity used for dedup. + logger.error("hardware digest: irreducible payload exceeds %d bytes", MAX_PAYLOAD_BYTES) + raise ValueError("hardware digest notification or scalar metadata exceeds payload ceiling") def _as_points(rows: Sequence[dict[str, Any]]) -> tuple[SeriesPoint, ...]: diff --git a/src/leapflow/hardware/observation.py b/src/leapflow/hardware/observation.py new file mode 100644 index 00000000..747d0fb1 --- /dev/null +++ b/src/leapflow/hardware/observation.py @@ -0,0 +1,566 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""Bounded operational observation, independent of command and task authority. + +One helper for policy validation, confirmation and bounded dispersion. Raw evidence +is never rewritten. Episode is also used by advisory reporting, so notification +backoff has one implementation and does not imply firmware protection. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import deque +from dataclasses import dataclass, field, replace +from statistics import median +from typing import Any, Mapping +from uuid import uuid4 + +from leapflow.hardware.context import Channel, as_numeric +from leapflow.hardware.transport import Reading + +DEFAULT_REPORT_AFTER_S = 60.0 +DEFAULT_REPORT_MAX_S = 900.0 +MAX_PENDING_TRANSITIONS = 32 + + +def calibration_identity(value: Mapping[str, Any]) -> str: + """Stable calibration scope from existing records or lifecycle telemetry.""" + if value.get("kind") == "invalidated" and value.get("event_id"): + return str(value["event_id"]) + if value.get("workcell_id") and value.get("generation") is not None: + # Generations are immutable within a workcell. Rollback telemetry carries + # the generation identity but need not repeat its model/dependency digests. + return json.dumps( + {key: value[key] for key in ("workcell_id", "generation")}, sort_keys=True, + ) + if value.get("procedure_id") and value.get("recorded_at") is not None: + return repr((value["procedure_id"], float(value["recorded_at"]))) + return str(value.get("event_id") or repr(dict(value))) + + +@dataclass(frozen=True) +class ObservationPolicy: + soft_min: float | None = None + soft_max: float | None = None + margin_floor: float | None = None + margin_max: float | None = None + confirm_s: float = 0.2 + recover_s: float = 0.5 + adaptive: bool = False + noise_window_s: float = 5.0 + min_fresh_samples: int = 32 + noise_multiplier: float = 3.0 + report_after_s: float = DEFAULT_REPORT_AFTER_S + report_max_s: float = DEFAULT_REPORT_MAX_S + + def __post_init__(self) -> None: + for name in self.__dataclass_fields__: + value = getattr(self, name) + if name == "adaptive": + if not isinstance(value, bool): + raise ValueError("observation.adaptive must be boolean") + continue + if value is None and name in {"soft_min", "soft_max", "margin_floor", "margin_max"}: + continue + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + raise ValueError(f"observation.{name} must be finite numeric data") + if name not in {"soft_min", "soft_max"} and value < 0: + raise ValueError(f"observation.{name} must not be negative") + if ( + self.soft_min is not None + and self.soft_max is not None + and self.soft_min >= self.soft_max + ): + raise ValueError("observation soft boundaries must be ordered") + if self.margin_max is not None and self.margin_max < (self.margin_floor or 0.0): + raise ValueError("observation.margin_max must not be below the floor") + if self.adaptive and self.margin_max is None: + raise ValueError("adaptive observation requires an authorized margin_max") + if not isinstance(self.min_fresh_samples, int) or not 4 <= self.min_fresh_samples <= 4096: + raise ValueError("observation.min_fresh_samples must be an integer in 4..4096") + if self.noise_window_s <= 0 or self.noise_multiplier <= 0 or self.report_after_s <= 0: + raise ValueError( + "observation window, multiplier and reminder interval must be positive" + ) + if self.report_max_s < self.report_after_s: + raise ValueError("observation.report_max_s must not be below report_after_s") + + def for_channel(self, channel: Channel) -> ObservationPolicy: + envelope = channel.envelope + floor = self.margin_floor + if floor is None: + floor = max(envelope.tolerance or 0.0, envelope.quantization or 0.0) + policy = replace(self, margin_floor=floor) + ceiling = policy.margin_max if policy.margin_max is not None else floor + if ceiling < floor: + raise ValueError( + f"observation ceiling below declared measurement floor on {channel.channel_id}" + ) + if policy.soft_min is None and policy.soft_max is None: + return policy + if ( + not channel.is_readable + or channel.is_media + or not envelope.declared + or envelope.allowed_values + ): + raise ValueError( + f"observation soft boundaries require a declared numeric reading: {channel.channel_id}" + ) + for soft, direction in ((policy.soft_min, -1), (policy.soft_max, 1)): + if soft is None: + continue + low, high = envelope.min_value, envelope.max_value + if low is not None and soft - ceiling < low: + raise ValueError( + f"observation threshold outside hard lower boundary: {channel.channel_id}" + ) + if high is not None and soft + ceiling > high: + raise ValueError( + f"observation threshold outside hard upper boundary: {channel.channel_id}" + ) + if direction == -1 and high is not None and soft + ceiling >= high: + raise ValueError("observation lower recovery band is collapsed") + if direction == 1 and low is not None and soft - ceiling <= low: + raise ValueError("observation upper recovery band is collapsed") + if policy.soft_min is not None and policy.soft_max is not None: + if policy.soft_min + ceiling >= policy.soft_max - ceiling: + raise ValueError("observation recovery bands must not collapse") + return policy + + +@dataclass(frozen=True) +class ObservationSettings: + defaults: ObservationPolicy = field(default_factory=ObservationPolicy) + overrides: tuple[tuple[str, str, ObservationPolicy], ...] = () + + @classmethod + def parse(cls, raw: Any) -> ObservationSettings: + if isinstance(raw, str): + try: + raw = json.loads(raw or "{}") + except json.JSONDecodeError as exc: + raise ValueError("hardware.observation must be valid JSON") from exc + if raw is None: + raw = {} + if not isinstance(raw, Mapping) or set(raw) - {"defaults", "overrides"}: + raise ValueError("hardware.observation accepts only defaults and exact overrides") + defaults = raw.get("defaults", {}) + if ( + not isinstance(defaults, Mapping) + or set(defaults) - ObservationPolicy.__dataclass_fields__.keys() + ): + raise ValueError("unknown observation default keys") + policy = ObservationPolicy(**defaults) + entries = raw.get("overrides", []) + if not isinstance(entries, (list, tuple)): + raise ValueError("observation.overrides must be an array") + overrides = [] + identities = set() + for entry in entries: + if not isinstance(entry, Mapping): + raise ValueError("observation override must be a mapping") + data = dict(entry) + device, channel = data.pop("device_id", None), data.pop("channel_id", None) + if not all( + isinstance(v, str) and v and not any(c in v for c in "*?[]") + for v in (device, channel) + ): + raise ValueError("observation override requires exact device_id and channel_id") + if (device, channel) in identities or set( + data + ) - ObservationPolicy.__dataclass_fields__.keys(): + raise ValueError("duplicate observation override or unknown keys") + identities.add((device, channel)) + overrides.append((device, channel, replace(policy, **data))) + return cls(policy, tuple(overrides)) + + def resolve(self, device_id: str, channel: Channel) -> ObservationPolicy: + policy = next( + (p for d, c, p in self.overrides if (d, c) == (device_id, channel.channel_id)), + self.defaults, + ) + return policy.for_channel(channel) + + def validate_contexts(self, contexts: Any) -> None: + admitted = {(ctx.device_id, ch.channel_id): ch for ctx in contexts for ch in ctx.channels} + for device, channel, _ in self.overrides: + if (device, channel) not in admitted: + raise ValueError( + f"observation override names an unadmitted channel: {device}.{channel}" + ) + for (device, _), channel in admitted.items(): + self.resolve(device, channel) + + +@dataclass +class Episode: + """Shared entry/recovery and exponential reminder lifecycle, no polling loop.""" + + active: bool = False + known: bool = False + since: float = 0.0 + next_report: float = 0.0 + interval: float = 0.0 + folded: int = 0 + episode: str = "" + sequence: int = 0 + candidate_at: float | None = None + recovering_at: float | None = None + + def unknown(self) -> None: + self.known = False + self.candidate_at = self.recovering_at = None + + def observe( + self, + *, + breached: bool, + clear: bool, + now: float, + confirm_s: float = 0.0, + recover_s: float = 0.0, + report_after_s: float = DEFAULT_REPORT_AFTER_S, + report_max_s: float = DEFAULT_REPORT_MAX_S, + ) -> str: + self.known = True + if not self.active: + self.recovering_at = None + if not breached: + self.candidate_at = None + return "" + if self.candidate_at is None: + self.candidate_at = now + if now - self.candidate_at + 1e-9 < confirm_s: + return "" + self.active, self.since = True, self.candidate_at + self.episode, self.sequence = uuid4().hex, 0 + self.interval, self.next_report = report_after_s, now + report_after_s + self.folded = 0 + return "entry" + self.candidate_at = None + if clear: + if self.recovering_at is None: + self.recovering_at = now + if now - self.recovering_at + 1e-9 >= recover_s: + self.active, self.recovering_at = False, None + self.sequence += 1 + return "recovery" + else: + self.recovering_at = None + if now >= self.next_report: + self.sequence += 1 + self.interval = min(self.interval * 2.0, report_max_s) + self.next_report, self.folded = now + self.interval, 0 + return "reminder" + self.folded += 1 + return "" + + +class ObservationJudgment: + """Per-channel state and notices. Rings supply bounded raw noise evidence.""" + + def __init__(self, device_id: str, channel: Channel, policy: ObservationPolicy) -> None: + self.device_id, self.channel, self.policy = device_id, channel, policy.for_channel(channel) + self.identity = hashlib.sha256(repr(channel.to_dict()).encode()).hexdigest()[:20] + self._declaration_identity = self.identity + self.runtime_id = uuid4().hex + self.states: dict[str, Episode] = {} + self.pending: dict[str, deque[dict[str, Any]]] = {} + self.latest: dict[str, dict[str, Any]] = {} + self.margin = float(self.policy.margin_floor or 0.0) + self._training_since: float | None = None + self._estimated_at = -math.inf + self._last_at: float | None = None + self._epoch = "" + self._noise_excess = False + self._noise_valid = False + self._revisions: dict[str, int] = {} + self._pending_omitted: dict[str, int] = {} + + def prepare_epoch(self, reading: Reading) -> None: + if reading.acquisition_epoch != self._epoch: + self.reconfigure(self.policy) + self._epoch = reading.acquisition_epoch + + def reconfigure(self, policy: ObservationPolicy) -> None: + self.policy = policy.for_channel(self.channel) + # Already confirmed evidence survives reload until the producer checkpoints it. + for name in ("soft_lower", "soft_upper", "observation_quality"): + if name in self.states: + self.states[name] = Episode() + self.invalidate() + + def reset_identity(self, identity: str) -> None: + changed = hashlib.sha256(f"{self._declaration_identity}/{identity}".encode()).hexdigest()[ + :20 + ] + if changed != self.identity: + self.identity = changed + self.reconfigure(self.policy) + + def invalidate(self) -> None: + for state in self.states.values(): + state.unknown() + self._training_since = None + self._estimated_at = -math.inf + self._last_at = None + if not self._margin_frozen: + self.margin = float(self.policy.margin_floor or 0.0) + self._noise_excess = False + self._noise_valid = False + + @property + def _margin_frozen(self) -> bool: + return any( + state.active + for name, state in self.states.items() + if name.startswith("soft_") or name == "observation_quality" + ) + + def condition( + self, + name: str, + *, + breached: bool, + clear: bool, + reading: Reading, + now: float, + valid: bool = True, + soft: bool = False, + reason: str = "", + extra: Mapping[str, Any] | None = None, + ) -> dict[str, Any] | None: + state = self.states.setdefault(name, Episode()) + policy = self.policy + phase = "" + if not valid: + state.unknown() + else: + phase = state.observe( + breached=breached, + clear=clear, + now=now, + confirm_s=policy.confirm_s if soft else 0.0, + recover_s=policy.recover_s if soft else 0.0, + report_after_s=policy.report_after_s, + report_max_s=policy.report_max_s, + ) + data = { + "key": f"{self.device_id}/{self.channel.channel_id}/{name}", + "condition": name, + "device_id": self.device_id, + "channel_id": self.channel.channel_id, + "identity": self.identity, + "runtime_id": self.runtime_id, + "episode": state.episode, + "sequence": state.sequence, + "active": state.active, + "known": state.known, + "phase": phase, + "value": as_numeric(reading.value), + "unit": self.channel.unit, + "reason": reason, + "duration_s": max(0.0, now - state.since) if state.episode else 0.0, + "observed_at": reading.observed_at, + "margin": self.margin, + "soft_min": policy.soft_min, + "soft_max": policy.soft_max, + "hard_min": self.channel.envelope.min_value, + "hard_max": self.channel.envelope.max_value, + "report_after_s": policy.report_after_s, + "report_max_s": policy.report_max_s, + } + data.update(extra or {}) + self.latest[name] = data + if not phase: + return None + notices = self.pending.setdefault(name, deque(maxlen=MAX_PENDING_TRANSITIONS)) + # Repeated reminders can be folded, but entry and recovery are distinct evidence. + if phase == "reminder" and notices and notices[-1]["phase"] == "reminder": + notices.pop() + revision = self._revisions.get(name, 0) + 1 + self._revisions[name] = revision + data["revision"] = revision + if len(notices) == MAX_PENDING_TRANSITIONS: + self._pending_omitted[name] = self._pending_omitted.get(name, 0) + 1 + notices.append(dict(data)) + return data + + def observe_soft( + self, + reading: Reading, + *, + ring: Any = None, + pending_motion: bool = False, + fault_free: bool = False, + ) -> list[dict[str, Any]]: + policy = self.policy + if policy.soft_min is None and policy.soft_max is None: + return [] + numeric = as_numeric(reading.value) + at = reading.acquisition_at if reading.acquisition_at is not None else reading.monotonic_at + valid = numeric is not None and reading.is_trustworthy and math.isfinite(at) + self.prepare_epoch(reading) + if self._last_at is not None and at <= self._last_at: + valid = False + if self._last_at is not None and self.channel.sample_rate_hz > 0: + if at - self._last_at > 2.0 / self.channel.sample_rate_hz: + self.invalidate() + if not valid: + self.invalidate() + return [] + self._last_at = at + notices = [] + for name, bound, direction in ( + ("soft_lower", policy.soft_min, -1), + ("soft_upper", policy.soft_max, 1), + ): + if bound is None: + continue + delta = direction * (numeric - bound) + notice = self.condition( + name, + breached=delta > self.margin, + clear=delta < -self.margin, + reading=reading, + now=at, + soft=True, + reason="continuous raw observation outside operational band", + ) + if notice: + notices.append(notice) + healthy = ( + not pending_motion + and fault_free + and self.channel.envelope.contains(numeric) + and all( + not s.active and s.candidate_at is None + for name, s in self.states.items() + if name != "observation_quality" + ) + and (policy.soft_min is None or numeric > policy.soft_min + self.margin) + and (policy.soft_max is None or numeric < policy.soft_max - self.margin) + ) + if not healthy: + self._training_since = None + self._noise_valid = False + elif policy.adaptive and ring is not None: + self._estimate(ring, at) + if policy.adaptive: + notice = self.condition( + "observation_quality", + breached=self._noise_excess, + clear=not self._noise_excess, + reading=reading, + now=at, + soft=True, + valid=self._noise_valid, + reason="healthy dispersion exceeds authorized margin ceiling", + ) + if notice: + notices.append(notice) + return notices + + def _estimate(self, ring: Any, at: float) -> None: + policy = self.policy + if self._training_since is None: + self._training_since = at + if at - self._training_since < self.channel.envelope.effective_settling_s: + return + if at - self._estimated_at < policy.noise_window_s / 4.0: + return + floor, ceiling = float(policy.margin_floor or 0.0), float(policy.margin_max or 0.0) + cutoff = max( + self._training_since + self.channel.envelope.effective_settling_s, + at - policy.noise_window_s, + ) + self._noise_valid = False + numbers: list[float] = [] + previous: float | None = None + for reading in ring: + stamp = ( + reading.acquisition_at + if reading.acquisition_at is not None + else reading.monotonic_at + ) + if stamp < cutoff: + continue + value = as_numeric(reading.value) + if ( + value is None + or not reading.is_trustworthy + or reading.acquisition_epoch != self._epoch + or not math.isfinite(stamp) + or ( + previous is not None + and ( + stamp <= previous + or ( + self.channel.sample_rate_hz > 0 + and stamp - previous > 2.0 / self.channel.sample_rate_hz + ) + ) + ) + ): + return + previous = stamp + if not self.channel.envelope.contains(value, margin=ceiling + floor): + return + numbers.append(value) + if len(numbers) < policy.min_fresh_samples: + if not self._margin_frozen: + self.margin = floor + self._noise_valid = False + return + center = median(numbers) + sigma = 1.4826 * median(abs(v - center) for v in numbers) + # A moving baseline is not random noise. No bias is inferred or compensated. + half = len(numbers) // 2 + trend = abs(median(numbers[:half]) - median(numbers[half:])) + if trend > max(floor, sigma): + self._training_since = at + if not self._margin_frozen: + self.margin = floor + return + self._estimated_at = at + self._noise_valid = True + requested = max(floor, policy.noise_multiplier * sigma) + self._noise_excess = requested > ceiling + # Dispersion can qualify quality recovery without retraining the margin + # during a warning episode. + if not self._margin_frozen: + self.margin = min(ceiling, requested) + + def acknowledge(self, checkpoints: Mapping[str, Mapping[str, Any]]) -> None: + """Discard only notices confirmed durable in the existing finding store.""" + for name, notices in self.pending.items(): + checkpoint = checkpoints.get(f"{self.device_id}/{self.channel.channel_id}/{name}", {}) + if checkpoint.get("runtime_id") != self.runtime_id: + continue + revision = checkpoint.get("revision", 0) + while notices and notices[0].get("revision", 0) <= revision: + notices.popleft() + + def snapshots(self) -> tuple[dict[str, Any], ...]: + rows = [] + for name, state in self.states.items(): + row = dict(self.latest.get(name, {})) + if not row: + continue + row.update( + active=state.active, + known=state.known, + identity=self.identity, + margin=self.margin, + pending_omitted=self._pending_omitted.get(name, 0), + pending=tuple(dict(n) for n in self.pending.get(name, ())), + ) + rows.append(row) + return tuple(rows) diff --git a/src/leapflow/hardware/registry.py b/src/leapflow/hardware/registry.py index cbf19a3c..ddffdc56 100644 --- a/src/leapflow/hardware/registry.py +++ b/src/leapflow/hardware/registry.py @@ -108,6 +108,7 @@ class HardwareSettings: hardware_alert_policies: Any = "" # JSON array of monitor-only advisory limits (see ``leapflow.hardware.advisory``). hardware_advisory_limits: Any = "" + hardware_observation: Any = "{}" # Profile-owned append-only audit destination for HardwareTools. audit_log_path: str = "" # Profile-owned record of which declarations an operator has reviewed, and @@ -149,6 +150,7 @@ def from_settings(cls, settings: Any) -> "HardwareSettings": enabled=bool(getattr(settings, "hardware_enabled", True)), hardware_alert_policies=getattr(settings, "hardware_alert_policies", ""), hardware_advisory_limits=getattr(settings, "hardware_advisory_limits", ""), + hardware_observation=getattr(settings, "hardware_observation", "{}"), ) return cls( enabled=bool(getattr(settings, "hardware_enabled", True)), @@ -193,6 +195,7 @@ def from_settings(cls, settings: Any) -> "HardwareSettings": fleet_nodes=str(getattr(settings, "hardware_fleet_nodes", "") or ""), hardware_alert_policies=getattr(settings, "hardware_alert_policies", ""), hardware_advisory_limits=getattr(settings, "hardware_advisory_limits", ""), + hardware_observation=getattr(settings, "hardware_observation", "{}"), audit_log_path=( str(profile_layout.hardware.audit_log_path) if profile_layout is not None else "" ), @@ -377,6 +380,10 @@ def __init__( providers: Sequence[HardwareContextProvider] = (), ) -> None: self._settings = settings or HardwareSettings() + from leapflow.hardware.observation import ObservationSettings + self._observation_settings = ObservationSettings.parse(self._settings.hardware_observation) + self._observation_health: dict[str, tuple[float, bool]] = {} + self._observation_identities: dict[str, str] = {} self._explicit_providers = tuple(providers) self._contexts: dict[str, HardwareContext] = {} # Declarations owned by external sources (fabric nodes), keyed by source id @@ -508,6 +515,9 @@ def _restore_command_recovery(self) -> None: f"committed generation {generation.generation} requires owner requalification and safety publication after restart" ) recovered[device_id] = (reason, generation, state.invalidation is not None) + from leapflow.hardware.observation import calibration_identity + basis = state.invalidation or {"workcell_id": state.workcell_id, **generation.basis()} + self.invalidate_observation(device_id, identity=calibration_identity(basis)) self._command_recovery = recovered self._command_recovery_error = "" except Exception: # noqa: BLE001 - unreadable state must never authorize commands @@ -649,6 +659,7 @@ def load(self) -> LoadReport: ) self._restore_command_recovery() + self._restore_observation_identities() if self._command_recovery_error: notes.append(AdmissionNote("", "physical_model_recovery", "warning", self._command_recovery_error)) else: @@ -1159,6 +1170,7 @@ async def halt(self, device_id: str) -> TransportStatus | None: socket holding frame claims -- and an instance dropped while unclosed kept that socket until the process exited. """ + halt_started = time.monotonic() lock = self._open_locks.setdefault(device_id, asyncio.Lock()) async with lock: transport = self._transports.pop(device_id, None) @@ -1179,6 +1191,10 @@ async def halt(self, device_id: str) -> TransportStatus | None: self._transports[device_id] = transport else: await self._retire(device_id, transport) + if status.stop_verified is True and self._node_sessions is not None: + self._node_sessions.settle_halt( + device_id, before=halt_started, evidence=status.detail or "verified halt", + ) return status async def _cache_if_still_open(self, device_id: str, transport: HardwareTransport) -> None: @@ -1228,6 +1244,105 @@ def opened_devices(self) -> tuple[str, ...]: # ── Streaming ── + def _restore_observation_identities(self) -> None: + """Read existing calibration identity at admission, never on the sample path.""" + from leapflow.hardware.observation import calibration_identity + self._observation_identities = { + device: identity for device, identity in self._observation_identities.items() + if device in self._contexts + } + path = self._settings.instrument_db_path + if not path or not Path(path).exists(): + return + store = self.calibration_store + for device_id in self._contexts: + if device_id in self._observation_identities: + continue + record = store.latest(device_id) + if record is not None: + self.invalidate_observation(device_id, identity=calibration_identity({ + "procedure_id": record.procedure_id, "recorded_at": record.recorded_at})) + + def observation_policy(self, device_id: str, channel: Any) -> Any: + """Resolve operational tuning without changing the admitted envelope.""" + return self._observation_settings.resolve(device_id, channel) + + def reconfigure_observation(self, raw: Any) -> None: + """Validate the complete update before replacing any running policy.""" + from dataclasses import replace + from leapflow.hardware.observation import ObservationSettings + policy = ObservationSettings.parse(raw) + policy.validate_contexts(self.contexts()) + resolved = [(detector, policy.resolve(detector._context.device_id, detector._channel)) + for source in self._stream_sources or () + for detector in source.observation_detectors] + for detector, channel_policy in resolved: + detector.reconfigure_observation(channel_policy) + self._observation_settings = policy + self._settings = replace(self._settings, hardware_observation=raw) + + def observation_conditions(self) -> tuple[dict[str, Any], ...]: + """Current detector state plus bounded unacknowledged transitions, no I/O.""" + return tuple(row for source in self._stream_sources or () + for detector in source.observation_detectors + for row in detector.observation_snapshots) + + def acknowledge_observation(self, checkpoints: Mapping[str, Mapping[str, Any]]) -> None: + """Retire delivered detector notices only after their finding is durable.""" + for source in self._stream_sources or (): + for detector in source.observation_detectors: + detector.acknowledge_observation(checkpoints) + + def note_observation(self, reading: Any) -> None: + """Cache explicit diagnostic fault evidence; unknown is never fault-free.""" + value = reading.value + if not isinstance(value, Mapping) or "receive_qualified" not in value: + return + motors = value.get("motors", ()) + diagnostics = list(motors) if isinstance(motors, (list, tuple)) else [] + if isinstance(value.get("gripper"), Mapping): + diagnostics.append(value["gripper"]) + faults = [flag for item in diagnostics if isinstance(item, Mapping) + and isinstance(item.get("faults"), Mapping) for flag in item["faults"].values()] + status = value.get("status", {}) + status_ok = (isinstance(status, Mapping) and status.get("error_code") == 0 + and status.get("arm_status") == 0) + status_fault = isinstance(status, Mapping) and any( + status.get(name) not in (None, 0) for name in ("error_code", "arm_status")) + faults_known = bool(diagnostics) and all( + isinstance(item, Mapping) and isinstance(item.get("faults"), Mapping) and bool(item["faults"]) + for item in diagnostics) + explicit = (value.get("fault_free") is True and not status_fault + and not any(flag is not False for flag in faults)) + qualified = explicit or ("fault_free" not in value and status_ok and faults_known + and all(flag is False for flag in faults)) + healthy = (reading.is_trustworthy and value.get("receive_qualified") is True + and value.get("settled") is True and qualified) + self._observation_health[reading.device_id] = (reading.monotonic_at, healthy) + + def observation_fault_free(self, device_id: str) -> bool: + evidence = self._observation_health.get(device_id) + if evidence is None or not evidence[1] or device_id in self._degradation_latches: + return False + context = self.context(device_id) + if context is None: + return False + deadline = max((2.0 / c.sample_rate_hz for c in context.streaming_channels if c.sample_rate_hz > 0), default=0.0) + return 0 <= time.monotonic() - evidence[0] <= deadline + + def invalidate_observation(self, device_id: str, *, identity: str = "") -> None: + self._observation_health.pop(device_id, None) + if identity and device_id in self._contexts: + self._observation_identities[device_id] = identity + for source in self._stream_sources or (): + for detector in source.observation_detectors: + if detector._context.device_id == device_id: + detector.invalidate_observation(identity=identity) + if self._alert_policy is not None: + context = self.context(device_id) + for channel in context.channels if context else (): + self._alert_policy.reset_channel(device_id, channel.channel_id) + def stream_sources(self) -> tuple[Any, ...]: """Return one signal source per streaming channel, building them once. @@ -1238,6 +1353,7 @@ def stream_sources(self) -> tuple[Any, ...]: if not self._settings.stream_enabled: return () if self._stream_sources is None: + self._observation_settings.validate_contexts(self.contexts()) from leapflow.hardware.alert_policy import build_alert_policy from leapflow.hardware.stream import build_batch_coordinators, build_stream_sources @@ -1263,6 +1379,11 @@ def stream_sources(self) -> tuple[Any, ...]: batch_device_ids=batch_device_ids, ) self._stream_sources = (*batch_coordinators, *individual_sources) + for source in self._stream_sources: + for detector in source.observation_detectors: + identity = self._observation_identities.get(detector._context.device_id) + if identity: + detector.invalidate_observation(identity=identity) return self._stream_sources @property @@ -1546,6 +1667,14 @@ def record_event(self, event: Any) -> None: off) is simply skipped. """ self._recent_events.append(event) + if getattr(event, "kind", "") in { + "calibration_completed", "calibration_activated", "calibration_rolled_back", "calibration_invalidated", + }: + from leapflow.hardware.observation import calibration_identity + value = getattr(event, "value", None) + identity = calibration_identity(value) if isinstance(value, Mapping) else repr( + (event.kind, getattr(event, "observed_at", time.time()))) + self.invalidate_observation(getattr(event, "device_id", ""), identity=identity) store = self._reading_store if store is None: return @@ -1965,6 +2094,7 @@ async def drop_transport(self, device_id: str) -> None: the close to finish instead of opening a second link beside the closing one -- a camera answers that second open with "device busy". """ + self.invalidate_observation(device_id) if device_id not in self._transports: return async with self._open_locks.setdefault(device_id, asyncio.Lock()): diff --git a/src/leapflow/hardware/skill_results.py b/src/leapflow/hardware/skill_results.py index dc1ec3d0..b04dd838 100644 --- a/src/leapflow/hardware/skill_results.py +++ b/src/leapflow/hardware/skill_results.py @@ -118,8 +118,10 @@ "is safe before running anything." ), FAILURE_DEVICE_BUSY: ( - "Another run or control session holds the arm. Wait for it (hw_skill_status) or " - "stop it (hw_skill_stop)." + "For an active run, wait (hw_skill_status) or stop it (hw_skill_stop). " + "For an unsettled physical effect, halt with hw_estop and confirm stop_verified=true " + "before resuming; stopping a run alone does not prove settlement. " + "Unsettled model transactions require owner reconciliation, not a halt." ), FAILURE_SKILL_UNAVAILABLE: ( "A requirement is unmet (the error names it). When result.readiness.next_steps is " diff --git a/src/leapflow/hardware/skills/runtime.py b/src/leapflow/hardware/skills/runtime.py index 8eacfb8f..6227af77 100644 --- a/src/leapflow/hardware/skills/runtime.py +++ b/src/leapflow/hardware/skills/runtime.py @@ -498,7 +498,7 @@ async def execute( # must still be refused before this runtime consumes the cached intention. sessions = getattr(self._registry, "node_sessions", None) if sessions is not None and (reserved := sessions.busy_device(spec.device_ids)): - return SkillOutcome.failed(FAILURE_DEVICE_BUSY, f"{reserved} held by a session, transaction or unsettled action") + return SkillOutcome.failed(FAILURE_DEVICE_BUSY, sessions.busy_message(reserved)) if self.cached_plan(digest) is not plan: return SkillOutcome.failed(FAILURE_PLAN_UNKNOWN, "plan expired or was executed while checking its model") busy = [device for device in spec.device_ids if device in self._held] diff --git a/src/leapflow/hardware/skills/workcell_reload.py b/src/leapflow/hardware/skills/workcell_reload.py index 8d3026bc..8d56dc42 100644 --- a/src/leapflow/hardware/skills/workcell_reload.py +++ b/src/leapflow/hardware/skills/workcell_reload.py @@ -78,8 +78,8 @@ def prepare(runtime: SkillRuntime, workcell: str) -> PreparedReload | SkillOutco if any(run.running for run in runtime._runs.values()) or runtime._held: return SkillOutcome.failed(FAILURE_DEVICE_BUSY, "reload requires all skill runs and unsettled holds to be idle") sessions = getattr(runtime._registry, "node_sessions", None) - if sessions is not None and sessions.busy_device(devices): - return SkillOutcome.failed(FAILURE_DEVICE_BUSY, "reload devices are held by a session or transaction") + if sessions is not None and (busy := sessions.busy_device(devices)): + return SkillOutcome.failed(FAILURE_DEVICE_BUSY, sessions.busy_message(busy)) return PreparedReload(workcell, previous, candidate, workcell_digest(previous), workcell_digest(candidate), runtime._node_session_id, devices) diff --git a/src/leapflow/hardware/stream.py b/src/leapflow/hardware/stream.py index 14ebb130..8ab6786a 100644 --- a/src/leapflow/hardware/stream.py +++ b/src/leapflow/hardware/stream.py @@ -11,25 +11,24 @@ exceeded, samples lost -- which occur at the rate at which something worth noticing actually happens. -Every detection rule is derived from the channel's own ``Envelope``. Nothing new is -declared: the limits a human already wrote down for approval are the same limits -that make an observation interesting. - -Two clocks are in play and they are not interchangeable. Intervals, slew rates and -staleness read ``Reading.monotonic_at``, because wall-clock jumps would fabricate -rates no device produced. Anything that leaves this module -- an event's timestamp, -a persisted window -- carries ``Reading.observed_at``, because every consumer -outside it (findings, audit, the board's time axis) is wall-clock. +Hard detection uses the channel's unchanged ``Envelope``. Report-only operational +warnings use the separately validated observation policy and never grant authority. + +Clocks are not interchangeable. Staleness uses host ``Reading.monotonic_at``; +physical rates require qualified acquisition intervals (remote host arrival timing +is unavailable for this purpose). Events and persisted windows carry wall-clock +``Reading.observed_at``, matching findings, audit, and the board's time axis. """ from __future__ import annotations import asyncio import logging +import math import time from collections import deque -from dataclasses import dataclass -from typing import Any, Callable, Deque, Iterable, Iterator, Protocol, runtime_checkable +from dataclasses import dataclass, field, replace +from typing import Any, Callable, Deque, Iterable, Iterator, Mapping, Protocol, runtime_checkable from leapflow.hardware.context import ( Channel, @@ -45,6 +44,8 @@ sensor_loss_debounce_s, ) from leapflow.hardware.transport import Reading +from leapflow.hardware.node_gate import rate_delta +from leapflow.hardware.observation import ObservationJudgment, ObservationPolicy, ObservationSettings logger = logging.getLogger(__name__) @@ -121,6 +122,8 @@ class HardwareEventClass: # breach informs a person and must never drive a reflex (see ``advisory``). "advisory_exceeded": HardwareEventClass.RECORD, "advisory_cleared": HardwareEventClass.RECORD, + "observation_warning": HardwareEventClass.RECORD, + "observation_cleared": HardwareEventClass.RECORD, } @@ -196,6 +199,8 @@ class EventKind: """A degradation policy triggered while actions are disabled; reported only.""" ADVISORY_EXCEEDED = "advisory_exceeded" """A monitor-only limit the device protects itself against was breached.""" + OBSERVATION_WARNING = "observation_warning" + OBSERVATION_CLEARED = "observation_cleared" ADVISORY_CLEARED = "advisory_cleared" """A breached advisory limit returned inside its hysteresis band.""" @@ -222,6 +227,9 @@ class HardwareEvent: it should fall back to ``classify_hardware_event(self.kind)``. """ + observation: dict[str, Any] = field(default_factory=dict) + sample_evidence: bool = False + @property def event_type(self) -> str: """Return the EventBus type. ``hw`` is the family every consumer groups on. @@ -278,6 +286,7 @@ def to_payload(self) -> dict[str, Any]: "unit": self.unit, "ts": self.observed_at, "_mono_ts": time.monotonic(), + "observation": dict(self.observation), } @@ -295,14 +304,23 @@ def __init__(self, capacity: int = DEFAULT_RING_CAPACITY) -> None: self._readings: Deque[Reading] = deque(maxlen=max(2, int(capacity))) self._dropped_sequences = 0 self._expected_sequence: int | None = None + self._sequence_epoch: str = "" + self._sequence_scope: str = "" def record(self, reading: Reading) -> int: """Append *reading*, returning how many samples appear to have been lost.""" lost = 0 - if self._expected_sequence is not None and reading.sequence > self._expected_sequence: - lost = reading.sequence - self._expected_sequence - self._dropped_sequences += lost - self._expected_sequence = reading.sequence + 1 + if (reading.acquisition_epoch, reading.sequence_scope) != (self._sequence_epoch, self._sequence_scope): + self._expected_sequence = None + self._sequence_epoch, self._sequence_scope = reading.acquisition_epoch, reading.sequence_scope + if reading.sequence_scope == "channel": + if self._expected_sequence is not None and reading.sequence > self._expected_sequence: + lost = reading.sequence - self._expected_sequence + self._dropped_sequences += lost + if self._expected_sequence is None or reading.sequence >= self._expected_sequence: + self._expected_sequence = reading.sequence + 1 + else: + self._expected_sequence = None self._readings.append(reading) return lost @@ -359,14 +377,15 @@ def summary(self) -> dict[str, Any]: class HardwareEventDetector: - """Turns a stream of readings into events, using only declared envelope data. + """Judge raw hard evidence and separately configured report-only observations. Stateful per channel: staleness and rate need to remember what came before. Kept separate from the ring so the detection rules can be tested against a synthetic series without a transport in the picture. """ - def __init__(self, context: HardwareContext, channel: Channel) -> None: + def __init__(self, context: HardwareContext, channel: Channel, + policy: ObservationPolicy | None = None) -> None: self._context = context self._channel = channel self._last_numeric: float | None = None @@ -374,12 +393,40 @@ def __init__(self, context: HardwareContext, channel: Channel) -> None: self._degraded_streak = 0 self._breached = False self._stale = False + self._last_acquisition: float | None = None + self._acquisition_epoch = "" + self._observation = ObservationJudgment(context.device_id, channel, policy or ObservationPolicy()) + self.safety_evidence: dict[str, HardwareEvent | None] = {} + + @property + def observation_snapshots(self) -> tuple[dict[str, Any], ...]: + return self._observation.snapshots() - def observe(self, reading: Reading, *, lost: int = 0) -> tuple[HardwareEvent, ...]: + def acknowledge_observation(self, checkpoints: Mapping[str, Mapping[str, Any]]) -> None: + self._observation.acknowledge(checkpoints) + + def reconfigure_observation(self, policy: ObservationPolicy) -> None: + if policy != self._observation.policy: + self._observation.reconfigure(policy) + + def invalidate_observation(self, *, identity: str = "") -> None: + self._last_numeric = self._last_acquisition = None + if identity: + self._observation.reset_identity(identity) + else: + self._observation.invalidate() + + def observe(self, reading: Reading, *, lost: int = 0, ring: ReadingRing | None = None, + pending_motion: bool = False, fault_free: bool = False) -> tuple[HardwareEvent, ...]: """Return the events this reading produces.""" events: list[HardwareEvent] = [] channel = self._channel envelope = channel.envelope + self._observation.prepare_epoch(reading) + self.safety_evidence = {EventKind.THRESHOLD_EXCEEDED: None, EventKind.STALE: None, + EventKind.RATE_EXCEEDED: None, EventKind.QUALITY_DEGRADED: None} + if reading.quality == Quality.STALE.value: + self.safety_evidence[EventKind.STALE] = self._event(EventKind.STALE, reading, "stale source evidence") if lost > 0: events.append( @@ -392,6 +439,8 @@ def observe(self, reading: Reading, *, lost: int = 0) -> tuple[HardwareEvent, .. if reading.quality != Quality.OK.value: self._degraded_streak += 1 + self.safety_evidence[EventKind.QUALITY_DEGRADED] = self._event( + EventKind.QUALITY_DEGRADED, reading, f"quality is {reading.quality!r}") # Three in a row rather than one: a lone suspect sample is noise, a run of # them is a fault. Reporting every one would reproduce the sampling-rate # flood this layer exists to prevent. @@ -409,10 +458,37 @@ def observe(self, reading: Reading, *, lost: int = 0) -> tuple[HardwareEvent, .. numeric = as_numeric(reading.value) if numeric is not None and envelope.declared: events.extend(self._numeric_events(reading, numeric)) - - self._last_numeric = numeric if numeric is not None else self._last_numeric - self._last_monotonic = reading.monotonic_at - self._stale = False + elif not reading.is_trustworthy or numeric is None: + self._observation.invalidate() + valid = numeric is not None and reading.is_trustworthy + acquisition = reading.acquisition_at if reading.acquisition_at is not None else reading.monotonic_at + continuous = (self._last_acquisition is None or acquisition > self._last_acquisition) + self._last_numeric = numeric if valid and continuous else None + self._last_acquisition = acquisition if valid and math.isfinite(acquisition) else None + self._acquisition_epoch = reading.acquisition_epoch + if reading.is_trustworthy and math.isfinite(reading.monotonic_at): + self._last_monotonic = reading.monotonic_at + notices = self._observation.observe_soft(reading, ring=ring, pending_motion=pending_motion, + fault_free=fault_free) + for notice in notices: + kind = EventKind.OBSERVATION_CLEARED if notice["phase"] == "recovery" else EventKind.OBSERVATION_WARNING + events.append(HardwareEvent(kind=kind, device_id=self._context.device_id, + channel_id=channel.channel_id, quantity=channel.quantity, + detail=(f"{notice['condition']}: {notice['reason']}; " + f"soft={notice['soft_min']}..{notice['soft_max']}, " + f"hard={notice['hard_min']}..{notice['hard_max']}, " + f"margin={notice['margin']:g}, duration={notice['duration_s']:.2f}s; reported only"), + value=reading.value, unit=channel.unit, observed_at=reading.observed_at, + observation=notice)) + # Bad numeric evidence invalidates soft/rate history, not the explicit + # quality/freshness condition carried by this same reading. + self._observation.condition("quality_degraded", breached=self._degraded_streak >= 3, + clear=reading.is_trustworthy, reading=reading, + now=reading.monotonic_at, reason="sampling quality") + self._observation.condition("stale", breached=reading.quality == Quality.STALE.value, + clear=reading.is_trustworthy, reading=reading, + now=reading.monotonic_at, reason="sampling freshness") + self._stale = not reading.is_trustworthy return tuple(events) def check_stale(self, *, now: float | None = None) -> tuple[HardwareEvent, ...]: @@ -428,24 +504,28 @@ def check_stale(self, *, now: float | None = None) -> tuple[HardwareEvent, ...]: never met, and in both cases the check reports nothing useful. """ channel = self._channel - if channel.sample_rate_hz <= 0 or self._last_monotonic is None or self._stale: + self.safety_evidence[EventKind.STALE] = None + if channel.sample_rate_hz <= 0 or self._last_monotonic is None: return () deadline = 2.0 / channel.sample_rate_hz elapsed = (now if now is not None else time.monotonic()) - self._last_monotonic if elapsed <= deadline: return () self._stale = True - return ( - HardwareEvent( - kind=EventKind.STALE, - device_id=self._context.device_id, - channel_id=channel.channel_id, - quantity=channel.quantity, - detail=(f"no sample for {elapsed:.2f}s on a {channel.sample_rate_hz:g} Hz channel"), - unit=channel.unit, - observed_at=time.time(), - ), - ) + self._last_numeric = self._last_acquisition = None + state = self._observation.states.get("stale") + if state is None or not state.active: + self._observation.invalidate() + reading = Reading(self._context.device_id, channel.channel_id, None, observed_at=time.time()) + notice = self._observation.condition("stale", breached=True, clear=False, reading=reading, + now=now if now is not None else time.monotonic(), reason="sampling silence") + event = HardwareEvent(kind=EventKind.STALE, device_id=self._context.device_id, + channel_id=channel.channel_id, quantity=channel.quantity, + detail=f"no sample for {elapsed:.2f}s on a {channel.sample_rate_hz:g} Hz channel", + unit=channel.unit, observed_at=reading.observed_at, + sample_evidence=True, observation=notice or {}) + self.safety_evidence[EventKind.STALE] = event + return (event,) if notice else () def _numeric_events(self, reading: Reading, numeric: float) -> list[HardwareEvent]: events: list[HardwareEvent] = [] @@ -466,29 +546,40 @@ def _numeric_events(self, reading: Reading, numeric: float) -> list[HardwareEven f"left the declared range ({_bounds(envelope)})", ) ) - elif envelope.contains(numeric, margin=envelope.settle_margin): + elif reading.is_trustworthy and envelope.contains(numeric, margin=envelope.settle_margin): # Re-entering is worth one event too, so a watcher can see recovery # instead of inferring it from silence. self._breached = False events.append(self._event(EventKind.SETTLED, reading, "returned to the declared range")) - if ( - envelope.max_rate is not None - and self._last_numeric is not None - and self._last_monotonic is not None - ): - elapsed = reading.monotonic_at - self._last_monotonic - if elapsed > 0 and envelope.rate_exceeded( - delta=numeric - self._last_numeric, elapsed_s=elapsed - ): - observed = abs(numeric - self._last_numeric) / elapsed - events.append( - self._event( - EventKind.RATE_EXCEEDED, - reading, - f"changing at {observed:g}/s, above the declared {envelope.max_rate:g}/s", - ) - ) + if not envelope.contains(numeric): + self.safety_evidence[EventKind.THRESHOLD_EXCEEDED] = self._event( + EventKind.THRESHOLD_EXCEEDED, reading, f"left the declared range ({_bounds(envelope)})") + at = reading.acquisition_at if reading.acquisition_at is not None else reading.monotonic_at + elapsed = at - self._last_acquisition if self._last_acquisition is not None else 0.0 + rate_valid = (reading.is_trustworthy and reading.rate_timebase != "unavailable" + and self._last_numeric is not None and math.isfinite(at) and elapsed > 0 + and reading.acquisition_epoch == self._acquisition_epoch + and (self._channel.sample_rate_hz <= 0 or elapsed <= 2.0 / self._channel.sample_rate_hz)) + delta = rate_delta(self._channel, numeric, self._last_numeric) if rate_valid else 0.0 + rate_breached = rate_valid and envelope.rate_exceeded(delta=delta, elapsed_s=elapsed) + if rate_breached: + event = self._event(EventKind.RATE_EXCEEDED, reading, + f"changing at {abs(delta) / elapsed:g}/s, above the declared {envelope.max_rate:g}/s") + self.safety_evidence[EventKind.RATE_EXCEEDED] = event + events.append(event) + now = reading.monotonic_at + threshold_notice = self._observation.condition( + "threshold_exceeded", breached=self._breached, clear=not self._breached, + reading=reading, now=now, valid=reading.is_trustworthy, + reason=f"declared range ({_bounds(envelope)})") + if threshold_notice: + events = [replace(event, observation=threshold_notice) + if event.kind in {EventKind.THRESHOLD_EXCEEDED, EventKind.SETTLED} else event + for event in events] + self._observation.condition("rate_exceeded", breached=bool(rate_breached), clear=not rate_breached, + reading=reading, now=now, valid=bool(rate_valid), reason="declared physical rate", + extra={"rate_available": bool(rate_valid)}) return events def _event(self, kind: str, reading: Reading, detail: str) -> HardwareEvent: @@ -501,9 +592,49 @@ def _event(self, kind: str, reading: Reading, detail: str) -> HardwareEvent: value=reading.value, unit=reading.unit or self._channel.unit, observed_at=reading.observed_at, + sample_evidence=kind in {EventKind.THRESHOLD_EXCEEDED, EventKind.RATE_EXCEEDED, EventKind.QUALITY_DEGRADED}, ) +def _stale_observation(detector: HardwareEventDetector, alert_policy: Any) -> tuple[HardwareEvent, ...]: + events = detector.check_stale() + if alert_policy is not None: + try: + alert_policy.evaluate_sample(detector._context.device_id, detector._channel.channel_id, + {EventKind.STALE: detector.safety_evidence.get(EventKind.STALE), + EventKind.THRESHOLD_EXCEEDED: None, EventKind.RATE_EXCEEDED: None, + EventKind.QUALITY_DEGRADED: None}) + except Exception: # noqa: BLE001 - a policy failure cannot stop acquisition + logger.exception("Hardware silence safety evaluation failed") + return events + + +def _observation_policy(registry: Any, context: HardwareContext, channel: Channel) -> ObservationPolicy: + resolver = getattr(registry, "observation_policy", None) + if callable(resolver): + return resolver(context.device_id, channel) + settings = getattr(registry, "settings", None) + return ObservationSettings.parse(getattr(settings, "hardware_observation", "{}")).resolve(context.device_id, channel) + + +def _observe_sample(detector: HardwareEventDetector, reading: Reading, ring: ReadingRing, + registry: Any, alert_policy: Any, *, lost: int) -> tuple[HardwareEvent, ...]: + note = getattr(registry, "note_observation", None) + if callable(note): + note(reading) + pending = getattr(registry, "pending_write_expectations", {}) + pending_motion = isinstance(pending, dict) and any(d == reading.device_id and v for (d, _), v in pending.items()) + evidence = getattr(registry, "observation_fault_free", None) + fault_free = bool(evidence(reading.device_id)) if callable(evidence) else False + events = detector.observe(reading, lost=lost, ring=ring, pending_motion=pending_motion, fault_free=fault_free) + if alert_policy is not None: + try: + alert_policy.evaluate_sample(reading.device_id, reading.channel_id, detector.safety_evidence) + except Exception: # noqa: BLE001 - report a policy failure without stopping acquisition + logger.exception("Hardware sample safety evaluation failed") + return events + + class HardwareStreamSource: """Samples one channel on a schedule, emitting derived events. @@ -532,7 +663,7 @@ def __init__( self._context = context self._channel = channel self.ring = ReadingRing(ring_capacity) - self._detector = HardwareEventDetector(context, channel) + self._detector = HardwareEventDetector(context, channel, _observation_policy(registry, context, channel)) self._event_sink = event_sink self._store = reading_store self._alert_policy = alert_policy @@ -552,6 +683,10 @@ def __init__( self._sensor_loss_triggered: bool = False self._last_degradation_outcome: Any = None + @property + def observation_detectors(self) -> tuple[HardwareEventDetector, ...]: + return (self._detector,) + @property def source_id(self) -> str: return f"hw:{self._context.device_id}:{self._channel.channel_id}" @@ -610,14 +745,15 @@ async def _run(self, emit: Any) -> None: # would otherwise produce log lines at the sampling rate, burying the # first and most useful one. if consecutive_failures == 1: + self._detector.invalidate_observation() logger.warning( "Hardware stream %s read failed: %s", self.source_id, exc, exc_info=True ) # Mark the start of a communication loss window. self._comm_loss_start = time.monotonic() + self._dispatch(_stale_observation(self._detector, self._alert_policy), emit) # DegradationPolicy: comm_loss enforcement. await self._maybe_trigger_comm_loss() - self._dispatch(self._detector.check_stale(), emit) await self._sleep(min(interval * (2**consecutive_failures), 30.0)) # Backoff deliberately abandons the old cadence; resuming the prior # deadline would burst to "catch up" on a device that just recovered. @@ -634,10 +770,11 @@ async def _run(self, emit: Any) -> None: consecutive_failures = 0 self._samples += 1 lost = self.ring.record(reading) + self._dispatch(_observe_sample(self._detector, reading, self.ring, self._registry, + self._alert_policy, lost=lost), emit) await self._persist(reading, lost=lost) # Sensor quality tracking for degradation policy. await self._track_sensor_quality(reading) - self._dispatch(self._detector.observe(reading, lost=lost), emit) next_at += interval delay = next_at - time.monotonic() @@ -732,7 +869,8 @@ async def _switch_sensor(self, target_channel_id: str) -> bool: if reading.quality != Quality.OK.value: return False self._channel = target - self._detector = HardwareEventDetector(self._context, target) + self._detector = HardwareEventDetector(self._context, target, + _observation_policy(self._registry, self._context, target)) return True @property @@ -793,11 +931,16 @@ def _dispatch(self, events: Iterable[HardwareEvent], emit: Any) -> None: the value and the unit are all needed downstream, and collapsing them to a detail string here would force every consumer to parse it back out. - Alert policy evaluation runs after the emit so the event is recorded (and - visible on the board) before a response is attempted. The policy itself - dispatches asynchronous tasks so it never blocks the sampling loop. + Sample-level safety evidence is evaluated before dispatch. Other hard events + are evaluated before presentation pacing; suppression cannot suppress a guard. + The policy dispatches asynchronous tasks and never blocks sampling. """ for event in events: + if self._alert_policy is not None and not event.sample_evidence: + try: + self._alert_policy.evaluate(event) + except Exception: # noqa: BLE001 - policy must not stop sampling + logger.exception("Hardware alert policy raised") if not self._admit(event): continue if self._event_sink is not None: @@ -810,16 +953,6 @@ def _dispatch(self, events: Iterable[HardwareEvent], emit: Any) -> None: emit(event) except Exception as exc: # noqa: BLE001 - as above logger.warning("Hardware event emit raised: %s", exc, exc_info=True) - # Alert policy evaluation: after emit so the event is visible first. - if self._alert_policy is not None: - try: - self._alert_policy.evaluate(event) - # Reset consecutive counters on recovery so a new breach cycle - # starts fresh rather than carrying stale counts. - if event.kind == EventKind.SETTLED: - self._alert_policy.reset_channel(event.device_id, event.channel_id) - except Exception as exc: # noqa: BLE001 - policy must not stop sampling - logger.warning("Hardware alert policy raised: %s", exc, exc_info=True) def _admit(self, event: HardwareEvent) -> bool: """Return whether this event clears the per-kind-per-channel rate floor. @@ -833,6 +966,8 @@ def _admit(self, event: HardwareEvent) -> bool: - The same kind on the *same* channel is still suppressed for the ``MIN_EVENT_INTERVAL_S`` floor, preventing level-triggered floods. """ + if event.observation.get("phase") in {"entry", "recovery", "reminder"}: + return True # episode transitions must survive notification pacing now = time.monotonic() key = f"{event.kind}:{event.device_id}.{event.channel_id}" previous = self._last_emitted.get(key) @@ -901,7 +1036,8 @@ def __init__( ch.channel_id: ReadingRing(ring_capacity) for ch in channels } self._detectors: dict[str, HardwareEventDetector] = { - ch.channel_id: HardwareEventDetector(context, ch) for ch in channels + ch.channel_id: HardwareEventDetector(context, ch, _observation_policy(registry, context, ch)) + for ch in channels } self._event_sink = event_sink self._store = reading_store @@ -926,6 +1062,10 @@ def __init__( self._failed_channel_id: str = "" self._last_degradation_outcome: Any = None + @property + def observation_detectors(self) -> tuple[HardwareEventDetector, ...]: + return tuple(self._detectors.values()) + @property def source_id(self) -> str: return f"hw.batch:{self._context.device_id}" @@ -977,15 +1117,17 @@ async def _run(self, emit: Any) -> None: except Exception as exc: # noqa: BLE001 - one device must not stop the rest consecutive_failures += 1 if consecutive_failures == 1: + for detector in self._detectors.values(): + detector.invalidate_observation() logger.warning( "Batch stream %s read failed: %s", self.source_id, exc, exc_info=True ) self._comm_loss_start = time.monotonic() + # Check staleness on all channels before an action can await I/O. + for detector in self._detectors.values(): + self._dispatch(_stale_observation(detector, self._alert_policy), emit) # DegradationPolicy: comm_loss enforcement. await self._maybe_trigger_comm_loss() - # Check staleness on all channels during failures. - for detector in self._detectors.values(): - self._dispatch(detector.check_stale(), emit) await self._sleep(min(interval * (2**consecutive_failures), 30.0)) next_at = time.monotonic() continue @@ -1000,19 +1142,26 @@ async def _run(self, emit: Any) -> None: consecutive_failures = 0 self._samples += 1 - # Distribute batch readings to per-channel rings and detectors. + # Judge every channel before any persistence await can delay a hard guard. + persisted = [] for reading in batch.readings: ch_id = reading.channel_id ring = self._rings.get(ch_id) if ring is None: continue lost = ring.record(reading) - await self._persist(reading, lost=lost) - # Sensor quality tracking for degradation policy. - await self._track_sensor_quality(reading) detector = self._detectors.get(ch_id) if detector is not None: - self._dispatch(detector.observe(reading, lost=lost), emit) + self._dispatch(_observe_sample(detector, reading, ring, self._registry, + self._alert_policy, lost=lost), emit) + persisted.append((reading, lost)) + received = {reading.channel_id for reading in batch.readings} + for ch_id, detector in self._detectors.items(): + if ch_id not in received: + self._dispatch(_stale_observation(detector, self._alert_policy), emit) + for reading, lost in persisted: + await self._persist(reading, lost=lost) + await self._track_sensor_quality(reading) next_at += interval delay = next_at - time.monotonic() @@ -1109,7 +1258,8 @@ async def _switch_sensor(self, target_channel_id: str) -> bool: self._channels = updated self._channel_ids = tuple(dict.fromkeys(channel.channel_id for channel in updated)) self._rings.setdefault(target_channel_id, ReadingRing(DEFAULT_RING_CAPACITY)) - self._detectors[target_channel_id] = HardwareEventDetector(self._context, target) + self._detectors[target_channel_id] = HardwareEventDetector( + self._context, target, _observation_policy(self._registry, self._context, target)) return True @property @@ -1153,6 +1303,11 @@ async def _sleep(self, seconds: float) -> None: def _dispatch(self, events: Iterable[HardwareEvent], emit: Any) -> None: """Hand events to the sink, signal pipeline, and alert policy, paced per kind.""" for event in events: + if self._alert_policy is not None and not event.sample_evidence: + try: + self._alert_policy.evaluate(event) + except Exception: # noqa: BLE001 - policy must not stop sampling + logger.exception("Hardware alert policy raised") if not self._admit(event): continue if self._event_sink is not None: @@ -1165,16 +1320,11 @@ def _dispatch(self, events: Iterable[HardwareEvent], emit: Any) -> None: emit(event) except Exception as exc: # noqa: BLE001 - as above logger.warning("Hardware event emit raised: %s", exc, exc_info=True) - if self._alert_policy is not None: - try: - self._alert_policy.evaluate(event) - if event.kind == EventKind.SETTLED: - self._alert_policy.reset_channel(event.device_id, event.channel_id) - except Exception as exc: # noqa: BLE001 - policy must not stop sampling - logger.warning("Hardware alert policy raised: %s", exc, exc_info=True) def _admit(self, event: HardwareEvent) -> bool: """Return whether this event clears the per-kind-per-channel rate floor.""" + if event.observation.get("phase") in {"entry", "recovery", "reminder"}: + return True # episode transitions must survive notification pacing now = time.monotonic() key = f"{event.kind}:{event.device_id}.{event.channel_id}" previous = self._last_emitted.get(key) diff --git a/src/leapflow/hardware/transport.py b/src/leapflow/hardware/transport.py index 41f75146..ffef4862 100644 --- a/src/leapflow/hardware/transport.py +++ b/src/leapflow/hardware/transport.py @@ -93,6 +93,19 @@ class Reading: monotonic_at: float = field(default_factory=time.monotonic) sequence: int = 0 quality: str = Quality.OK.value + sequence_scope: str = "channel" + """Only contiguous channel acquisition sequences certify dropped samples. + + Poll/read counters and device-wide counters must declare another scope. + """ + acquisition_at: float | None = None + acquisition_epoch: str = "" + """Source-local interval evidence, compared only within one source epoch. + + This counter is never compared with the host clock or used for wall time. + """ + rate_timebase: str = "local" + """Local intervals are usable; remote arrival timing alone is unavailable.""" @property def is_trustworthy(self) -> bool: @@ -114,6 +127,9 @@ def to_dict(self) -> dict[str, Any]: "observed_at": self.observed_at, "sequence": self.sequence, "quality": self.quality, + "sequence_scope": self.sequence_scope, + "acquisition_at": self.acquisition_at, + "acquisition_epoch": self.acquisition_epoch, } diff --git a/src/leapflow/hardware/transports/host.py b/src/leapflow/hardware/transports/host.py index b4ce4d92..404dbbe9 100644 --- a/src/leapflow/hardware/transports/host.py +++ b/src/leapflow/hardware/transports/host.py @@ -118,6 +118,7 @@ async def read(self, channel_id: str) -> Reading: quantity=channel.quantity if channel is not None else "", unit=channel.unit if channel is not None else "", sequence=sequence, + sequence_scope="channel_read", # A probe that cannot answer right now is suspect, not zero. Reporting a # value would put a fabricated number into the downsample window, where # nothing downstream could ever tell it apart from a measurement. diff --git a/src/leapflow/hardware/transports/lhp.py b/src/leapflow/hardware/transports/lhp.py index 52e35e18..1490df15 100644 --- a/src/leapflow/hardware/transports/lhp.py +++ b/src/leapflow/hardware/transports/lhp.py @@ -721,6 +721,9 @@ def _localize_reading(self, payload: Mapping[str, Any], channel_id: str) -> Read failure_code="lhp_result_mismatch", ) observed = _as_positive_float(payload.get("observed_at")) + acquisition = _as_positive_float(payload.get("acquisition_at")) + epoch = payload.get("acquisition_epoch") + epoch = epoch if isinstance(epoch, str) else "" return Reading( device_id=self._local_device_id(), channel_id=channel_id, @@ -730,6 +733,10 @@ def _localize_reading(self, payload: Mapping[str, Any], channel_id: str) -> Read observed_at=observed if observed > 0 else time.time(), sequence=int(payload.get("sequence") or 0), quality=str(payload.get("quality") or Quality.OK.value), + sequence_scope=str(payload.get("sequence_scope") or "unknown"), + acquisition_at=acquisition if acquisition > 0 and epoch else None, + acquisition_epoch=epoch, + rate_timebase="source" if acquisition > 0 and epoch else "unavailable", ) def _frame_from_payload(self, payload: Any, channel_id: str) -> FrameReading: @@ -832,7 +839,7 @@ def _as_positive_float(value: Any) -> float: if isinstance(value, bool) or not isinstance(value, (int, float)): return 0.0 numeric = float(value) - return numeric if numeric > 0 else 0.0 + return numeric if math.isfinite(numeric) and numeric > 0 else 0.0 def _latest_observed_at(readings: list[Reading]) -> float: diff --git a/src/leapflow/hardware/transports/media.py b/src/leapflow/hardware/transports/media.py index b89ba82c..cbfde274 100644 --- a/src/leapflow/hardware/transports/media.py +++ b/src/leapflow/hardware/transports/media.py @@ -232,6 +232,7 @@ async def read(self, channel_id: str) -> Reading: quantity=channel.quantity, unit=channel.unit, sequence=self._sequence, + sequence_scope="device_read", quality=Quality.OK.value, ) diff --git a/src/leapflow/hardware/transports/mock.py b/src/leapflow/hardware/transports/mock.py index 9ed707f1..7258b700 100644 --- a/src/leapflow/hardware/transports/mock.py +++ b/src/leapflow/hardware/transports/mock.py @@ -132,6 +132,7 @@ async def read(self, channel_id: str) -> Reading: quantity=channel.quantity if channel is not None else "", unit=channel.unit if channel is not None else "", sequence=sequence, + sequence_scope="channel_read", quality=Quality.OK.value, ) diff --git a/src/leapflow/hardware/transports/model_service.py b/src/leapflow/hardware/transports/model_service.py index 5aa080a9..79e0d484 100644 --- a/src/leapflow/hardware/transports/model_service.py +++ b/src/leapflow/hardware/transports/model_service.py @@ -167,6 +167,7 @@ async def read(self, channel_id: str) -> Reading: value=value, quantity="model_metadata", sequence=self._sequence, + sequence_scope="device_read", quality=quality, ) diff --git a/src/leapflow/hardware/transports/robot_arm.py b/src/leapflow/hardware/transports/robot_arm.py index d5ce9dc7..a83af513 100644 --- a/src/leapflow/hardware/transports/robot_arm.py +++ b/src/leapflow/hardware/transports/robot_arm.py @@ -159,6 +159,7 @@ async def read(self, channel_id: str) -> Reading: quantity=ch.quantity, unit=ch.unit, sequence=seq, + sequence_scope="channel_read", quality=Quality.OK.value, ) @@ -336,6 +337,7 @@ async def read_batch(self, channel_ids: tuple[str, ...]) -> BatchReading: observed_at=now_wall, monotonic_at=now_mono, sequence=seq, + sequence_scope="channel_read", quality=Quality.OK.value, ) ) diff --git a/src/leapflow/hardware/transports/ros2.py b/src/leapflow/hardware/transports/ros2.py index d69e3116..ba7b7b65 100644 --- a/src/leapflow/hardware/transports/ros2.py +++ b/src/leapflow/hardware/transports/ros2.py @@ -263,6 +263,7 @@ async def read(self, channel_id: str) -> Reading: channel_id=channel_id, value=None, sequence=self._next_seq(channel_id), + sequence_scope="channel_read", quality=Quality.STALE.value, ) @@ -285,6 +286,7 @@ async def read(self, channel_id: str) -> Reading: quantity=ch.quantity if ch else "", unit=ch.unit if ch else "", sequence=self._next_seq(channel_id), + sequence_scope="channel_read", quality=quality, ) diff --git a/src/leapflow/monitor/finding_store.py b/src/leapflow/monitor/finding_store.py index f4876253..17279311 100644 --- a/src/leapflow/monitor/finding_store.py +++ b/src/leapflow/monitor/finding_store.py @@ -110,6 +110,32 @@ def exists_dedup(self, watch_id: str, dedup_key: str) -> bool: ).fetchone() return row is not None + def latest_notification_checkpoints(self, domain: str) -> dict[str, dict[str, Any]]: + """Latest persisted condition checkpoint across watch recreation, no new table. + + Ordinary dedup remains all-time per watch. Only producers carrying explicit + notification metadata participate in this query. + """ + rows = self._con.execute( + """ + SELECT payload FROM ( + SELECT payload, row_number() OVER ( + PARTITION BY json_extract_string(payload, '$.notification.key') + ORDER BY ts DESC, rowid DESC + ) AS position + FROM monitor_findings + WHERE domain = ? AND json_extract_string(payload, '$.notification.key') IS NOT NULL + ) WHERE position = 1 + """, [domain], + ).fetchall() + checkpoints = {} + for (payload,) in rows: + data = self._safe_json(payload, {}) + checkpoint = data.get("notification", {}) if isinstance(data, dict) else {} + if isinstance(checkpoint, dict) and checkpoint.get("key"): + checkpoints[str(checkpoint["key"])] = checkpoint + return checkpoints + def delete_for_watch(self, watch_id: str) -> None: """Remove all findings belonging to a watch.""" execute_with_retry( diff --git a/src/leapflow/tools/config_tools.py b/src/leapflow/tools/config_tools.py index 1d6c3c15..3cd17921 100644 --- a/src/leapflow/tools/config_tools.py +++ b/src/leapflow/tools/config_tools.py @@ -88,7 +88,7 @@ def _service() -> Any: """Build a ConfigService over the active settings.""" from leapflow.config_service import ConfigService - return ConfigService(_active_settings()) + return ConfigService(_active_settings(), hardware_registry=getattr(_context_ref, "_hardware_registry", None)) def _field_payload(view: Any, *, include_description: bool = True) -> Dict[str, Any]: diff --git a/src/leapflow/version.py b/src/leapflow/version.py index c40c3552..d2e71224 100644 --- a/src/leapflow/version.py +++ b/src/leapflow/version.py @@ -1,4 +1,4 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Version information for leapflow.""" -__version__ = "0.8.0+main" +__version__ = "0.8.1+main" diff --git a/tests/mock_signals/generators.py b/tests/mock_signals/generators.py index 73cf790e..2b0968a0 100644 --- a/tests/mock_signals/generators.py +++ b/tests/mock_signals/generators.py @@ -423,19 +423,10 @@ def _make_reading(self, ch: HardwareChannelSpec) -> Dict[str, Any]: if random.random() < ch.quality_degradation_rate: quality = random.choice(_DEGRADED_QUALITIES) - return { - "device_id": self.device_id, - "channel_id": ch.channel_id, - "value": value, - "quantity": ch.quantity, - "unit": ch.unit, - "observed_at": time.time(), - "sequence": seq, - "quality": quality, - # Deliberately included for test-pipeline monotonic ordering - # even though Reading.to_dict() omits it for persistence. - "monotonic_at": time.monotonic(), - } + from leapflow.hardware.transport import Reading + reading = Reading(device_id=self.device_id, channel_id=ch.channel_id, value=value, + quantity=ch.quantity, unit=ch.unit, sequence=seq, quality=quality) + return {**reading.to_dict(), "monotonic_at": reading.monotonic_at} def _make_hw_event( self, ch: HardwareChannelSpec, kind: str @@ -447,21 +438,11 @@ def _make_hw_event( detail = self._event_detail(ch, kind, value) # Import at function level to avoid circular dependency. - from leapflow.hardware.stream import classify_hardware_event + from leapflow.hardware.stream import HardwareEvent - return { - "kind": kind, - "event_class": classify_hardware_event(kind), - "source": f"{self.device_id}.{ch.channel_id}", - "device_id": self.device_id, - "channel_id": ch.channel_id, - "quantity": ch.quantity, - "detail": detail, - "value": value, - "unit": ch.unit, - "ts": now_wall, - "_mono_ts": time.monotonic(), - } + return HardwareEvent(kind=kind, device_id=self.device_id, channel_id=ch.channel_id, + quantity=ch.quantity, detail=detail, value=value, + unit=ch.unit, observed_at=now_wall).to_payload() @staticmethod def _event_detail( diff --git a/tests/test_config_service.py b/tests/test_config_service.py index 0d098aae..4f3e5d2f 100644 --- a/tests/test_config_service.py +++ b/tests/test_config_service.py @@ -333,3 +333,29 @@ def test_written_file_has_0600_permissions(self, tmp_path: Path) -> None: mode = stat.S_IMODE(target.stat().st_mode) assert mode == 0o600 + + +def test_hardware_observation_is_discoverable_validated_and_round_tripped(config_env) -> None: + from leapflow.config import load_config + settings, service = config_env + assert "hardware.observation" in service.writable_keys() + devices = settings.profile_layout.hardware.devices_dir + devices.mkdir(parents=True, exist_ok=True) + _write_yaml_atomic(devices / "bench.yaml", { + "hc_version": "hc.v0", "device_id": "bench", "transport": {"kind": "mock"}, + "channels": [{"channel_id": "level", "direction": "read", "quantity": "q", + "envelope": {"declared": True, "min_value": 0, "max_value": 100}}], + }) + valid = json.dumps({"overrides": [{"device_id": "bench", "channel_id": "level", "soft_max": 80, + "margin_floor": 1, "margin_max": 5, "adaptive": True}]}) + assert service.describe("hardware.observation").hot_reload == "yes" + result = service.set("hardware.observation", valid) + assert result.ok and result.warnings == () + assert load_config().hardware_observation == valid + invalid = json.dumps({"overrides": [{"device_id": "bench", "channel_id": "level", "soft_max": 100, + "margin_floor": 1}]}) + with pytest.raises(ValueError): + service.set("hardware.observation", invalid) + assert load_config().hardware_observation == valid + with pytest.raises(ValueError): + service.set("hardware.observation", '{"defaults":{"unknown":true}}') diff --git a/tests/test_hardware_advisory.py b/tests/test_hardware_advisory.py index 13f7f10d..d936a270 100644 --- a/tests/test_hardware_advisory.py +++ b/tests/test_hardware_advisory.py @@ -111,18 +111,22 @@ def test_active_advisories_follow_the_newest_event_per_limit(): assert active_advisories(rows) == ["arm.temp"] -def test_producer_marks_an_active_advisory_notable_and_a_cleared_one_info(): +def test_producer_assesses_current_conditions_without_inferencing_health_from_event_history(): from leapflow.hardware.observability.producer import _assess from leapflow.monitor.types import Severity def digest(*kinds): events = tuple({"kind": k, "device_id": "arm", "channel_id": "t", "quantity": "temp"} for k in kinds) - return SimpleNamespace(events=events, series=(), storage={}, sampling=(), devices=(1,)) + return SimpleNamespace(events=events, conditions=(), series=(), storage={}, sampling=(), devices=(1,)) severity, headline = _assess(digest(ADVISORY_EXCEEDED)) - assert severity == Severity.NOTABLE and "arm.temp" in headline + assert severity == Severity.INFO and "unknown" in headline.lower() assert _assess(digest(ADVISORY_CLEARED, ADVISORY_EXCEEDED))[0] == Severity.INFO + current = digest() + current.conditions = ({"key": "arm.temp", "condition": "advisory:temp", "active": True, "known": True},) + severity, headline = _assess(current) + assert severity == Severity.NOTABLE and "arm.temp" in headline @pytest.mark.parametrize("raw,count", [ diff --git a/tests/test_hardware_alert_and_observability.py b/tests/test_hardware_alert_and_observability.py index a7cb97d4..a8c62242 100644 --- a/tests/test_hardware_alert_and_observability.py +++ b/tests/test_hardware_alert_and_observability.py @@ -590,7 +590,7 @@ def reset_channel(self, device_id: str, channel_id: str) -> None: assert len(evaluated) == 1 assert evaluated[0].kind == EventKind.THRESHOLD_EXCEEDED - def test_dispatch_resets_on_settled(self) -> None: + def test_dispatch_settled_does_not_reset_unrelated_safety_evidence(self) -> None: resets = [] class _Policy: @@ -640,7 +640,7 @@ def reset_channel(self, device_id: str, channel_id: str) -> None: observed_at=time.time(), ) source._dispatch([settled_event], None) - assert ("dev", "ch") in resets + assert resets == [] # healthy raw samples reset each relevant condition independently # ════════════════════════════════════════════════════════════════ diff --git a/tests/test_hardware_governance.py b/tests/test_hardware_governance.py index 3dfcc09f..b55cc8a6 100644 --- a/tests/test_hardware_governance.py +++ b/tests/test_hardware_governance.py @@ -1891,6 +1891,7 @@ def test_hardware_config_keys_are_discoverable() -> None: "hardware.stream_enabled", "hardware.stream_ring_capacity", "hardware.persist_readings", + "hardware.observation", "hardware.downsample_interval_s", "hardware.raw_retention_days", "hardware.history_retention_days", @@ -1940,9 +1941,9 @@ def test_hardware_config_keys_are_discoverable() -> None: for key in keys: view = service.describe(key) assert view.description, f"{key} has no description in the catalog" - # Enabling hardware composes the approval classifier at construction, so it - # cannot take effect on a running daemon. - assert view.hot_reload == "restart-required" + # Observation tuning updates existing detectors; hardware construction + # settings still require a restart. + assert view.hot_reload == ("yes" if key == "hardware.observation" else "restart-required") def test_explicitly_disabled_hardware_leaves_the_classifier_untouched() -> None: diff --git a/tests/test_hardware_observability.py b/tests/test_hardware_observability.py index 2cfce32d..7e9dd783 100644 --- a/tests/test_hardware_observability.py +++ b/tests/test_hardware_observability.py @@ -784,6 +784,13 @@ def test_producer_severity_matches_digest_event_severity() -> None: ) +def test_report_only_observation_warning_is_notable_and_clear_is_info() -> None: + from leapflow.hardware.observability.digest import _event_severity + + assert _event_severity("observation_warning") == "notable" + assert _event_severity("observation_cleared") == "info" + + def test_digest_payload_clock_is_wall() -> None: """Every digest payload must declare ``clock=='wall'``. @@ -850,3 +857,171 @@ def walk(node): assert preview["param_from_row"]["channel"] == "preview_channel", ( "Preview must name the actual previewable channel, so it can start the target panel" ) + + +def _condition_registry(*, hard: bool = False) -> Any: + from leapflow.hardware.context import Channel, Envelope + from leapflow.hardware.observation import ObservationJudgment, ObservationPolicy + channel = Channel(channel_id="uncharted", direction="read", quantity="q", sample_rate_hz=10, + envelope=Envelope(declared=True, min_value=0, max_value=100)) + judgment = ObservationJudgment("bench", channel, ObservationPolicy(soft_max=80, margin_floor=1)) + class Conditions(_Registry): + def observation_conditions(self) -> Any: + return judgment.snapshots() + registry = Conditions() + registry.judgment = judgment + registry.condition_name = "threshold_exceeded" if hard else "soft_upper" + return registry + + +def _transition(registry: Any, value: float, at: float, *, valid: bool = True) -> None: + from leapflow.hardware.transport import Reading + name = registry.condition_name + registry.judgment.condition(name, breached=value > 81, clear=value < 79, + reading=Reading("bench", "uncharted", value, observed_at=_WALL + at), now=at, + valid=valid, soft=name.startswith("soft")) + + +@pytest.mark.parametrize("hard", [False, True]) +def test_episode_notifications_persist_across_watch_recreation_and_chart_churn(tmp_path: Any, hard: bool) -> None: + from leapflow.monitor.finding_store import FindingStore + store = FindingStore(tmp_path / "findings.duckdb") + registry = _condition_registry(hard=hard) + producer = HardwareObservationProducer(lambda: registry, finding_store=store) + def poll(now: float, watch: str = "w1") -> list[Any]: + ctx = _producer_ctx(_WALL + now) + ctx.spec.watch_id = watch + return list(asyncio.run(producer.observe(ctx))) + try: + _transition(registry, 83, 0) + _transition(registry, 83, .2) + first = poll(1) + assert len(first) == 1 + assert first[0].severity.value == ("alert" if hard else "notable") + store.save(first[0]) + registry._events = tuple(_event("sample_loss", observed_at=_WALL + i) for i in range(60)) + registry._windows = _windows(40) + quiet = poll(2) + assert all(f.severity.value == "info" for f in quiet) + recreated = HardwareObservationProducer(lambda: registry, finding_store=store) + ctx = _producer_ctx(_WALL + 3) + ctx.spec.watch_id = "w2" + assert all(f.severity.value == "info" for f in asyncio.run(recreated.observe(ctx))) + _transition(registry, 50, 4) + _transition(registry, 50, 4.5) + recovery = poll(5, "w2") + assert recovery[0].payload["notification"]["phase"] == "recovery" + store.save(recovery[0]) + _transition(registry, 83, 6) + _transition(registry, 83, 6.2) + recurrence = poll(7, "w2") + assert recurrence[0].dedup_key != first[0].dedup_key + assert recurrence[0].payload["notification"]["phase"] == "entry" + assert first[0].payload["notification"]["channel_id"] == "uncharted" + assert "raw=83" in first[0].summary + finally: + store.close() + + +def test_hard_entry_and_recovery_between_polls_are_both_reportable_once(tmp_path: Any) -> None: + from leapflow.monitor.finding_store import FindingStore + store = FindingStore(tmp_path / "findings.duckdb") + registry = _condition_registry(hard=True) + producer = HardwareObservationProducer(lambda: registry, finding_store=store) + try: + _transition(registry, 101, 0) + _transition(registry, 50, .1) + findings = asyncio.run(producer.observe(_producer_ctx(_WALL + 120))) + assert [f.payload["notification"]["phase"] for f in findings] == ["entry", "recovery"] + for finding in findings: + store.save(finding) + assert store.latest_notification_checkpoints("hardware")["bench/uncharted/threshold_exceeded"]["active"] is False + assert all(f.severity.value == "info" for f in asyncio.run(producer.observe(_producer_ctx(_WALL + 121)))) + finally: + store.close() + + +def test_failed_persistence_retries_the_same_episode(tmp_path: Any) -> None: + from leapflow.monitor.finding_store import FindingStore + store = FindingStore(tmp_path / "findings.duckdb") + registry = _condition_registry() + producer = HardwareObservationProducer(lambda: registry, finding_store=store) + try: + _transition(registry, 83, 0) + _transition(registry, 83, .2) + first = asyncio.run(producer.observe(_producer_ctx(_WALL + 1)))[0] + retried = asyncio.run(producer.observe(_producer_ctx(_WALL + 2)))[0] + assert retried.dedup_key == first.dedup_key + store.save(retried) + assert asyncio.run(producer.observe(_producer_ctx(_WALL + 3)))[0].severity.value == "info" + finally: + store.close() + + +def test_restart_waits_for_fresh_confirmation_and_preserves_reminder_progress(tmp_path: Any) -> None: + from leapflow.monitor.finding_store import FindingStore + store = FindingStore(tmp_path / "findings.duckdb") + registry = _condition_registry() + producer = HardwareObservationProducer(lambda: registry, finding_store=store) + try: + _transition(registry, 83, 0) + _transition(registry, 83, .2) + entry = asyncio.run(producer.observe(_producer_ctx(_WALL + 1)))[0] + store.save(entry) + registry = _condition_registry() # new runtime, no restored healthy assumption + producer = HardwareObservationProducer(lambda: registry, finding_store=store) + assert asyncio.run(producer.observe(_producer_ctx(_WALL + 2)))[0].severity.value == "info" + _transition(registry, 83, 2) + assert asyncio.run(producer.observe(_producer_ctx(_WALL + 2)))[0].severity.value == "info" + _transition(registry, 83, 2.2) + assert asyncio.run(producer.observe(_producer_ctx(_WALL + 3)))[0].severity.value == "info" + _transition(registry, 83, 62.2) + reminder = asyncio.run(producer.observe(_producer_ctx(_WALL + 63)))[0] + metadata = reminder.payload["notification"] + assert metadata["phase"] == "reminder" + assert metadata["episode"] == entry.payload["notification"]["episode"] + assert metadata["sequence"] == 1 and metadata["interval"] == 120 + store.save(reminder) + _transition(registry, 83, 182.2) + assert asyncio.run(producer.observe(_producer_ctx(_WALL + 182)))[0].severity.value == "info" + next_reminder = asyncio.run(producer.observe(_producer_ctx(_WALL + 184)))[0] + assert next_reminder.payload["notification"]["interval"] == 240 + finally: + store.close() + + +def test_condition_presentation_bound_does_not_limit_monitoring() -> None: + import json + from leapflow.hardware.observability.series import HardwareDigest, MAX_PAYLOAD_BYTES + rows = tuple({"key": f"device/channel/{i}", "reason": "x" * 1024} for i in range(1000)) + payload = HardwareDigest(generated_at=_WALL, conditions=rows).to_payload() + assert payload["counts"]["conditions"] == len(rows) + assert payload["conditions_omitted"] > 0 + assert len(json.dumps(payload, ensure_ascii=False).encode()) <= MAX_PAYLOAD_BYTES + + +def test_notification_payload_bounds_preserve_checkpoint_and_do_not_mutate_base() -> None: + import copy + import json + from leapflow.hardware.observability.series import HardwareDigest, MAX_PAYLOAD_BYTES, _fit + rows = tuple({"reason": "x" * 4000} for _ in range(100)) + base = HardwareDigest(generated_at=_WALL, events=rows, calibration=rows, conditions=rows).to_payload() + original = copy.deepcopy(base) + checkpoint = {"key": "device/channel/soft_upper", "episode": "e1", "sequence": 2, + "reason": "x" * 8192} + payload = _fit({**base, "notification": checkpoint}) + assert payload["notification"] == checkpoint + assert base == original + assert len(json.dumps(payload, ensure_ascii=False).encode()) <= MAX_PAYLOAD_BYTES + assert any(payload.get(f"{key}_omitted", 0) for key in ("conditions", "events", "calibration")) + + +def test_historical_advisory_events_notify_once_without_claiming_current_health() -> None: + registry = _Registry(events=(_event("advisory_exceeded"),)) + producer = HardwareObservationProducer(lambda: registry) + first = asyncio.run(producer.observe(_producer_ctx(_WALL + 1))) + assert first[0].severity.value == "notable" + assert first[0].payload["notification"]["phase"] == "entry" + quiet = asyncio.run(producer.observe(_producer_ctx(_WALL + 2))) + assert all(f.severity.value == "info" for f in quiet) + assert "unknown" in quiet[0].title.lower() diff --git a/tests/test_hardware_stream.py b/tests/test_hardware_stream.py index 4ece09d7..2c33304a 100644 --- a/tests/test_hardware_stream.py +++ b/tests/test_hardware_stream.py @@ -228,8 +228,8 @@ def test_returning_to_range_is_reported_as_recovery() -> None: def test_rate_event_uses_the_declared_max_rate() -> None: detector = _detector(max_rate=5.0) detector.observe(_reading(10.0, sequence=1, at=1.0)) - # 40 units in one second, against a declared 5/s. - events = detector.observe(_reading(50.0, sequence=2, at=2.0)) + # Continuous 10 Hz acquisition: 4 units in 0.1 seconds against 5/s. + events = detector.observe(_reading(14.0, sequence=2, at=1.1)) assert EventKind.RATE_EXCEEDED in [e.kind for e in events] @@ -820,3 +820,533 @@ def test_step_settling_default_unchanged_through_detector() -> None: detector = HardwareEventDetector(context, context.channel("level")) assert detector._channel.envelope.settling_model == "step" assert detector._channel.envelope.effective_settling_s == 0.0 + + +@pytest.mark.parametrize("raw", [ + {"defaults": {"adaptive": True}}, + {"defaults": {"margin_floor": -1}}, + {"defaults": {"soft_max": float("nan")}}, + {"defaults": {"confirm_s": True}}, + {"defaults": {"min_fresh_samples": 32.0}}, + {"defaults": {"unexpected": 1}}, + {"overrides": [{"device_id": "*", "channel_id": "level"}]}, +]) +def test_observation_rejects_invalid_policy(raw: Any) -> None: + from leapflow.hardware.observation import ObservationSettings + with pytest.raises(ValueError): + ObservationSettings.parse(raw) + + +def test_fixed_soft_observation_confirmation_recovery_and_recurrence() -> None: + from leapflow.hardware.observation import ObservationPolicy + context = _context() + detector = HardwareEventDetector(context, context.channel("level"), + ObservationPolicy(soft_min=10, soft_max=80, margin_floor=2)) + def sample(value: float, index: int) -> tuple[HardwareEvent, ...]: + return detector.observe(_reading(value, sequence=index, at=1 + index * 0.1)) + assert not sample(83, 0) + assert not sample(83, 1) + entry = sample(83, 2) + assert [e.kind for e in entry] == [EventKind.OBSERVATION_WARNING] + assert entry[0].resolved_event_class == "record" + assert sample(79, 3) == () # neither entering nor clearing the band + for index in range(4, 9): + assert sample(77, index) == () + clear = sample(77, 9) + assert [e.kind for e in clear] == [EventKind.OBSERVATION_CLEARED] + assert not sample(83, 10) + assert not sample(83, 11) + recurrence = sample(83, 12) + assert recurrence[0].observation["episode"] != entry[0].observation["episode"] + assert context.channel("level").envelope.max_value == 100 + + +def test_soft_confirmation_resets_on_invalid_and_missing_evidence() -> None: + from leapflow.hardware.observation import ObservationPolicy + context = _context() + detector = HardwareEventDetector(context, context.channel("level"), + ObservationPolicy(soft_max=80, margin_floor=2)) + detector.observe(_reading(83, sequence=0, at=1)) + detector.observe(_reading(83, sequence=1, at=1.1, quality="stale")) + for index in range(2): + assert EventKind.OBSERVATION_WARNING not in [e.kind for e in detector.observe( + _reading(83, sequence=2 + index, at=1.2 + 0.1 * index))] + assert EventKind.OBSERVATION_WARNING in [e.kind for e in detector.observe(_reading(83, sequence=4, at=1.4))] + detector.check_stale(now=2) + soft = next(row for row in detector.observation_snapshots if row["condition"] == "soft_upper") + assert soft["active"] and not soft["known"] + assert all(n["phase"] != "recovery" for n in soft["pending"]) + + +def test_observation_channel_validation_and_atomic_reload() -> None: + context = _context() + registry = _registry(context) + sources = registry.stream_sources() + valid = '{"overrides":[{"device_id":"sampled_device","channel_id":"level","soft_max":80,"margin_floor":2}]}' + registry.reconfigure_observation(valid) + policy = sources[0].observation_detectors[0]._observation.policy + assert policy.soft_max == 80 + for raw in ('{"defaults":{"soft_max":100,"margin_floor":2}}', + '{"overrides":[{"device_id":"missing","channel_id":"level"}]}'): + with pytest.raises(ValueError): + registry.reconfigure_observation(raw) + assert sources[0].observation_detectors[0]._observation.policy == policy + assert registry.settings.hardware_observation == valid + + +def test_global_poll_sequences_do_not_claim_channel_loss() -> None: + from dataclasses import replace + ring = ReadingRing() + assert ring.record(replace(_reading(1, sequence=1), sequence_scope="device_read")) == 0 + assert ring.record(replace(_reading(1, sequence=99), sequence_scope="device_read")) == 0 + assert ring.dropped == 0 + assert ring.record(replace(_reading(1, sequence=100), acquisition_epoch="reconnect")) == 0 + + +def test_reordered_or_repeated_channel_sequences_do_not_invent_loss() -> None: + ring = ReadingRing() + for sequence in (1, 2, 1, 3, 3, 4): + assert ring.record(_reading(50, sequence=sequence)) == 0 + assert ring.dropped == 0 + assert ring.record(_reading(50, sequence=6)) == 1 + + +def test_circular_rates_and_discontinuous_history() -> None: + from dataclasses import replace + context = _context(max_rate=50) + channel = replace(context.channel("level"), unit="deg", envelope=Envelope( + declared=True, min_value=-180, max_value=180, max_rate=50)) + detector = HardwareEventDetector(context, channel) + detector.observe(_reading(179, sequence=1, at=1)) + assert EventKind.RATE_EXCEEDED not in [e.kind for e in detector.observe(_reading(-179, sequence=2, at=1.1))] + detector.observe(_reading(None, sequence=3, at=1.2)) + assert EventKind.RATE_EXCEEDED not in [e.kind for e in detector.observe(_reading(0, sequence=4, at=1.3))] + detector.observe(_reading(10, sequence=5, at=1.3)) + assert EventKind.RATE_EXCEEDED not in [e.kind for e in detector.observe(_reading(100, sequence=6, at=1.4))] + + +def _adaptive_replay(values: list[float], *, adaptive: bool = True, ceiling: float = 8, + pending_motion: bool = False, fault_free: bool = True) -> tuple[Any, list[Any]]: + from leapflow.hardware.observation import ObservationPolicy + context = _context(sample_rate_hz=100) + detector = HardwareEventDetector(context, context.channel("level"), ObservationPolicy( + soft_min=10, soft_max=80, margin_floor=1, margin_max=ceiling, adaptive=adaptive, + noise_window_s=1, min_fresh_samples=32)) + ring = ReadingRing(128) + events = [] + for index, value in enumerate(values): + reading = _reading(value, sequence=index + 1, at=1 + index * .01) + ring.record(reading) + events.extend(detector.observe(reading, ring=ring, pending_motion=pending_motion, fault_free=fault_free)) + return detector, events + + +@pytest.mark.parametrize("amplitude", [0, .1, 1]) +def test_synthetic_in_tolerance_replay_has_no_false_transitions(amplitude: float) -> None: + values = [50 + amplitude * (-1, 0, 1)[index % 3] for index in range(400)] + for adaptive in (False, True): + detector, events = _adaptive_replay(values, adaptive=adaptive) + assert not events + assert 1 <= detector._observation.margin <= 8 + if adaptive and amplitude == 1: + assert detector._observation.margin == pytest.approx(3 * 1.4826) + + +def test_spikes_sustained_excursion_recovery_and_drift_on_identical_replays() -> None: + values = [50.] * 100 + [90.] + [50.] * 10 + [83.] * 40 + [70.] * 80 + for adaptive in (False, True): + _, events = _adaptive_replay(values, adaptive=adaptive) + warnings = [e for e in events if e.kind == EventKind.OBSERVATION_WARNING] + clears = [e for e in events if e.kind == EventKind.OBSERVATION_CLEARED] + assert len(warnings) == len(clears) == 1 + assert warnings[0].observation["duration_s"] == pytest.approx(.2) + assert warnings[0].value == 83 # baseline shift never moves the operational boundary + + +def test_adaptive_margin_freezes_during_warning_and_resets_on_reconnect() -> None: + from dataclasses import replace + detector, _ = _adaptive_replay([[49., 50., 51.][i % 3] for i in range(200)]) + frozen = detector._observation.margin + ring = ReadingRing() + for index in range(40): + reading = _reading(86., sequence=201 + index, at=3 + index * .01) + ring.record(reading) + detector.observe(reading, ring=ring, fault_free=True) + assert detector._observation.margin == frozen + assert detector._observation.states["soft_upper"].active + detector.invalidate_observation() + assert not detector._observation.states["soft_upper"].known + assert detector._observation.margin == frozen + healthy, _ = _adaptive_replay([[49., 50., 51.][i % 3] for i in range(200)]) + healthy.observe(replace(_reading(50., sequence=1, at=4), acquisition_epoch="new-source"), + ring=ReadingRing(), fault_free=True) + assert healthy._observation.margin == 1 + + +@pytest.mark.parametrize("pending_motion,fault_free", [(True, True), (False, False)]) +def test_unqualified_or_pending_motion_never_trains_noise(pending_motion: bool, fault_free: bool) -> None: + detector, _ = _adaptive_replay([[49., 50., 51.][i % 3] for i in range(200)], pending_motion=pending_motion, fault_free=fault_free) + assert detector._observation.margin == 1 + assert detector._observation._training_since is None + + +def test_rising_noise_quality_warning_is_bounded_and_missing_data_is_not_recovery() -> None: + detector, events = _adaptive_replay([[46., 50., 54.][i % 3] for i in range(400)], ceiling=5) + quality = [e for e in events if e.observation.get("condition") == "observation_quality"] + assert len(quality) == 1 and quality[0].kind == EventKind.OBSERVATION_WARNING + assert detector._observation.margin == 5 + detector.observe(_reading(None, sequence=401, at=5, quality="stale")) + state = detector._observation.states["observation_quality"] + assert state.active and not state.known + assert not any(n["phase"] == "recovery" for n in detector._observation.pending["observation_quality"]) + + +def test_quality_episode_freezes_margin_through_unknown_and_recovery() -> None: + detector, _ = _adaptive_replay([[46., 50., 54.][i % 3] for i in range(400)], ceiling=5) + judgment = detector._observation + assert judgment.states["observation_quality"].active + detector.invalidate_observation() + assert judgment.margin == 5 + ring = ReadingRing(128) + recovery = None + for index in range(300): + reading = _reading(50, sequence=401 + index, at=5 + index * .01) + ring.record(reading) + events = detector.observe(reading, ring=ring, fault_free=True) + if recovery is None: + assert judgment.margin == 5 + for event in events: + if (event.kind == EventKind.OBSERVATION_CLEARED + and event.observation["condition"] == "observation_quality"): + recovery = event + assert event.observation["margin"] == 5 + assert recovery is not None + assert not judgment.states["observation_quality"].active + assert judgment.margin == 1 # retraining only after the episode has recovered + + +def test_sustained_trend_and_hard_boundary_proximity_never_relax_margin() -> None: + trend, _ = _adaptive_replay([50 + index * .05 for index in range(400)]) + assert trend._observation.margin == 1 + boundary, _ = _adaptive_replay([1., 2.] * 100) + assert boundary._observation.margin == 1 + + +def test_consecutive_hard_safety_uses_samples_not_breach_entry_events() -> None: + from leapflow.hardware.alert_policy import AlertRule, HardwareAlertPolicy + from leapflow.hardware.stream import _observe_sample + context = _context(max_rate=5) + detector = HardwareEventDetector(context, context.channel("level")) + policy = HardwareAlertPolicy([AlertRule(event_kind="threshold_exceeded", action="hw_estop", require_consecutive=3)]) + ring = ReadingRing() + emitted = [] + for index, value in enumerate([101, 101, 101, 101, 50, 101, 101, 101]): + reading = _reading(value, sequence=index + 1, at=1 + index * .1) + lost = ring.record(reading) + emitted.extend(_observe_sample(detector, reading, ring, None, policy, lost=lost)) + if index == 2: + assert policy.fired_count == 1 + assert policy.fired_count == 2 + assert sum(e.kind == EventKind.THRESHOLD_EXCEEDED for e in emitted) == 2 + + +def test_record_only_observation_cannot_be_mapped_to_reflex_action() -> None: + from leapflow.hardware.alert_policy import AlertRule, HardwareAlertPolicy + policy = HardwareAlertPolicy([AlertRule(event_kind="observation_warning", action="hw_estop")]) + policy.evaluate(HardwareEvent(kind="observation_warning", device_id="d", channel_id="c", quantity="q", detail="reported only")) + assert policy.fired_count == 0 + + +def test_true_hard_recurrence_survives_presentation_pacing() -> None: + context = _context() + detector = HardwareEventDetector(context, context.channel("level")) + source = HardwareStreamSource(None, context, context.channel("level")) + emitted = [] + for index, value in enumerate([101, 50, 101, 50]): + source._dispatch(detector.observe(_reading(value, sequence=index + 1, at=1 + index * .1)), emitted.append) + assert [e.kind for e in emitted] == ["threshold_exceeded", "settled"] * 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("adaptive", [False, True]) +async def test_scalar_batch_sampling_policy_parity(adaptive: bool) -> None: + from types import SimpleNamespace + from leapflow.hardware.stream import BatchStreamCoordinator + from leapflow.hardware.observation import ObservationSettings + values = [49., 50., 51.] * 40 + [90.] * 10 + [50.] * 20 + context = _context(sample_rate_hz=1000) + tuning = ObservationSettings.parse({"overrides": [{"device_id": context.device_id, "channel_id": "level", + "soft_max": 80, "margin_floor": 1, "margin_max": 8, "adaptive": adaptive, + "noise_window_s": .05, "confirm_s": .002, "recover_s": .005}]}) + outputs = [] + for batch_mode in (False, True): + class ReplayIO: + index = 0 + def device_io(self, device_id: str) -> Any: + return asyncio.Lock() + async def transport(self, device_id: str) -> Any: + return self + def observation_policy(self, device_id: str, channel: Any) -> Any: + return tuning.resolve(device_id, channel) + def observation_fault_free(self, device_id: str) -> bool: + return True + async def read(self, channel_id: str) -> Reading: + index = self.index + self.index += 1 + if self.index == len(values): + source._stopping.set() + return _reading(values[index], sequence=index + 1, at=1 + index * .001) + async def read_batch(self, device_id: str, channel_ids: Any) -> Any: + return SimpleNamespace(readings=(await self.read("level"),)) + io = ReplayIO() + source = (BatchStreamCoordinator(io, context, (context.channel("level"),)) if batch_mode + else HardwareStreamSource(io, context, context.channel("level"))) + events = [] + await asyncio.wait_for(source._run(events.append), timeout=2) + outputs.append([(e.kind, e.value, e.observation.get("duration_s")) for e in events]) + assert outputs[0] == outputs[1] + assert [row[0] for row in outputs[0]] == ["observation_warning", "observation_cleared"] + + +async def _sampling_cadence_probe(path: Any, *, adaptive: bool = False, factory: Any = HardwareStreamSource) -> dict[str, Any]: + """Offline cadence probe through real acquisition, buffering, and DuckDB writes.""" + import json + + context = _context(sample_rate_hz=500) + registry = _registry(context, instrument_db_path=str(path), downsample_interval_s=1, + hardware_observation=json.dumps({"overrides": [{ + "device_id": context.device_id, "channel_id": "level", "soft_max": 80, + "margin_floor": 1, "margin_max": 8, "adaptive": adaptive, + }]})) + store = registry.reading_store + source = factory(registry, context, context.channel("level"), reading_store=store) + events = [] + try: + await source.start(events.append) + await asyncio.sleep(2.2) + await source.stop() + return {**source.health, "windows": store.windows_written, "events": len(events)} + finally: + await source.stop() + await registry.close_all() + registry._instrument_conn.close() + + +@pytest.mark.asyncio +async def test_observation_modes_preserve_scheduled_acquisition_and_persistence(tmp_path: Any) -> None: + for adaptive in (False, True): + result = await _sampling_cadence_probe(tmp_path / f"cadence-{adaptive}.duckdb", adaptive=adaptive) + assert result["windows"] > 0 + assert result["events"] == 0 + # Leave headroom for shared CI scheduling; this is a regression guard, + # not qualification of a physical device's maximum acquisition rate. + assert result["observed_hz"] >= result["declared_hz"] * .8 + + +def test_stale_level_evidence_fires_before_notification_reminders(monkeypatch: Any) -> None: + from leapflow.hardware.alert_policy import AlertRule, HardwareAlertPolicy + from leapflow.hardware.stream import _stale_observation + context = _context() + detector = HardwareEventDetector(context, context.channel("level")) + policy = HardwareAlertPolicy([AlertRule(event_kind="stale", action="hw_estop", require_consecutive=3)]) + detector.observe(_reading(50, sequence=1, at=1)) + monkeypatch.setattr("leapflow.hardware.stream.time.monotonic", lambda: 1.3) + assert [e.kind for e in _stale_observation(detector, policy)] == ["stale"] + assert not _stale_observation(detector, policy) + assert not _stale_observation(detector, policy) + assert policy.fired_count == 1 + assert [e.observation["phase"] for e in detector.check_stale(now=61.3)] == ["reminder"] + detector.observe(_reading(50, sequence=2, at=62)) + assert not detector._observation.states["stale"].active + assert detector._observation.pending["stale"][-1]["phase"] == "recovery" + + +def test_sampling_silence_breaks_other_consecutive_safety_runs(monkeypatch: Any) -> None: + from leapflow.hardware.alert_policy import AlertRule, HardwareAlertPolicy + from leapflow.hardware.stream import _observe_sample, _stale_observation + context = _context() + detector = HardwareEventDetector(context, context.channel("level")) + policy = HardwareAlertPolicy([AlertRule( + event_kind="threshold_exceeded", action="hw_estop", require_consecutive=3)]) + ring = ReadingRing() + for index in range(2): + reading = _reading(101, sequence=index + 1, at=1 + index * .1) + _observe_sample(detector, reading, ring, None, policy, lost=ring.record(reading)) + monkeypatch.setattr("leapflow.hardware.stream.time.monotonic", lambda: 1.2) + _stale_observation(detector, policy) # acquisition failed, even before stale deadline + assert policy.fired_count == 0 + for index in range(3): + reading = _reading(101, sequence=3 + index, at=1.3 + index * .1) + _observe_sample(detector, reading, ring, None, policy, lost=ring.record(reading)) + assert policy.fired_count == (1 if index == 2 else 0) + + +def test_missing_numeric_values_do_not_hide_explicit_stale_quality() -> None: + context = _context() + detector = HardwareEventDetector(context, context.channel("level")) + detector.observe(_reading(None, sequence=1, at=1, quality="stale")) + state = detector._observation.states["stale"] + assert state.active and state.known + assert detector.safety_evidence[EventKind.STALE] is not None + detector.observe(_reading({"status": "fresh"}, sequence=2, at=1.1)) + assert not state.active and state.known + + +def test_calibration_identity_reset_preserves_notices_and_is_idempotent() -> None: + from leapflow.hardware.observation import ObservationPolicy + context = _context() + detector = HardwareEventDetector(context, context.channel("level"), ObservationPolicy( + soft_max=80, margin_floor=1, confirm_s=0)) + detector.observe(_reading(83, sequence=1, at=1)) + before = detector.observation_snapshots + old_identity = detector._observation.identity + detector.invalidate_observation(identity="qualified-generation-2") + snapshot = next(row for row in detector.observation_snapshots if row["condition"] == "soft_upper") + assert not snapshot["known"] and not snapshot["active"] + assert snapshot["identity"] != old_identity + assert snapshot["pending"][0]["identity"] == old_identity + assert snapshot["pending"][0] == next(row for row in before if row["condition"] == "soft_upper")["pending"][0] + detector.observe(_reading(83, sequence=2, at=1.1)) + episode = detector._observation.states["soft_upper"].episode + detector.invalidate_observation(identity="qualified-generation-2") + assert detector._observation.states["soft_upper"].episode == episode + + +@pytest.mark.component +def test_persisted_calibration_identity_is_initialized_without_restoring_health(tmp_path: Any) -> None: + from leapflow.hardware.calibration_store import CalibrationRecord + from leapflow.hardware.observation import ObservationPolicy + path = str(tmp_path / "instrument.duckdb") + context = _context() + registry = _registry(context, instrument_db_path=path) + record = CalibrationRecord(context.device_id, "zero", 123, {"qualified": "software test"}) + try: + assert registry.calibration_store.record(record) + registry.record_event(HardwareEvent( + kind="calibration_completed", device_id=context.device_id, channel_id="level", + quantity="q", detail="software test", value={ + "procedure_id": record.procedure_id, "recorded_at": record.recorded_at})) + detector = registry.stream_sources()[0].observation_detectors[0] + before = detector._observation.identity + detector.reconfigure_observation(ObservationPolicy(soft_max=80, margin_floor=1, confirm_s=0)) + detector.observe(_reading(83, sequence=1, at=1)) + assert detector._observation.states["soft_upper"].active + finally: + registry._instrument_conn.close() + restarted = _registry(context, instrument_db_path=path) + try: + fresh = restarted.stream_sources()[0].observation_detectors[0] + assert fresh._observation.identity == before + assert fresh._observation.margin == 0 + assert fresh.observation_snapshots == () + finally: + restarted._instrument_conn.close() + + +@pytest.mark.component +@pytest.mark.parametrize("kind", ["activated", "invalidated", "rolled_back"]) +def test_generation_identity_matches_lifecycle_evidence_after_restart(tmp_path: Any, kind: str) -> None: + from leapflow.hardware.calibration.lifecycle import project_decision + context = _context() + path = str(tmp_path / "instrument.duckdb") + registry = _registry(context, instrument_db_path=path) + try: + store = registry.calibration_generations + generation = store.commit_generation( + workcell_id="cell", request_id="software-test", expected_generation=None, + dependency_digest="deps", model={"device_id": context.device_id, "synthetic": True}, + accepted_blocks={"test": {"synthetic": True}}, causal={"software_test": True}) + if kind == "invalidated": + store.invalidate(workcell_id="cell", request_id="invalidate-test", + expected_generation=generation.generation, dependency_digest="deps", + causal={"software_test": True}) + elif kind == "rolled_back": + store.commit_generation( + workcell_id="cell", request_id="second-software-test", expected_generation=1, + dependency_digest="deps", model={"device_id": context.device_id, "synthetic": 2}, + accepted_blocks={"test": {"synthetic": True}}, causal={"software_test": True}) + store.rollback(workcell_id="cell", request_id="rollback-test", expected_generation=2, + dependency_digest="deps", causal={"software_test": True}) + decision = next(event for event in store.pending_events() if event["kind"] == kind) + registry.record_event(HardwareEvent( + kind=f"calibration_{decision['kind']}", device_id=context.device_id, + channel_id="calibration_lifecycle", quantity="q", detail="software test", + value=project_decision(decision, context.device_id))) + before = registry.stream_sources()[0].observation_detectors[0]._observation.identity + finally: + registry._instrument_conn.close() + restarted = _registry(context, instrument_db_path=path) + try: + detector = restarted.stream_sources()[0].observation_detectors[0] + assert detector._observation.identity == before + assert detector.observation_snapshots == () + finally: + restarted._instrument_conn.close() + + +def test_notice_overflow_is_bounded_disclosed_and_durable_notices_are_retired() -> None: + from leapflow.hardware.observation import MAX_PENDING_TRANSITIONS + context = _context() + detector = HardwareEventDetector(context, context.channel("level")) + for index in range(40): + detector.observe(_reading(101 if index % 2 == 0 else 50, sequence=index + 1, at=1 + index * .1)) + row = next(row for row in detector.observation_snapshots if row["condition"] == "threshold_exceeded") + assert len(row["pending"]) == MAX_PENDING_TRANSITIONS + assert row["pending_omitted"] == 40 - MAX_PENDING_TRANSITIONS + registry = _registry(context) + registered = registry.stream_sources()[0].observation_detectors[0] + registered.observe(_reading(101, sequence=1, at=1)) + notice = registered._observation.pending["threshold_exceeded"][0] + registry.acknowledge_observation({notice["key"]: {**notice, "runtime_id": "another-runtime"}}) + assert registered._observation.pending["threshold_exceeded"] + registry.acknowledge_observation({notice["key"]: notice}) + assert not registered._observation.pending["threshold_exceeded"] + assert registered._observation.states["threshold_exceeded"].active + + +@pytest.mark.parametrize("patch,expected", [ + ({}, True), ({"settled": False}, False), ({"receive_qualified": False}, False), + ({"fault_free": False}, False), ({"status": {"error_code": 1}}, False), + ({"motors": [{"faults": {"overheat": True}}]}, False), + ({"motors": [{"faults": {"overheat": None}}]}, False), +]) +def test_noise_training_requires_complete_fresh_fault_and_settled_evidence(patch: Any, expected: bool, + monkeypatch: Any) -> None: + from dataclasses import replace + registry = _registry(_context()) + value = {"receive_qualified": True, "settled": True, "fault_free": True, **patch} + reading = _reading(value, sequence=1, at=1) + monkeypatch.setattr("leapflow.hardware.registry.time.monotonic", lambda: 1.1) + registry.note_observation(reading) + assert registry.observation_fault_free("sampled_device") is expected + registry.note_observation(replace(reading, monotonic_at=2)) + assert not registry.observation_fault_free("sampled_device") + registry.note_observation(replace(reading, monotonic_at=0)) + assert not registry.observation_fault_free("sampled_device") + + +@pytest.mark.asyncio +async def test_batch_judges_every_hard_guard_before_persistence() -> None: + from dataclasses import replace + from types import SimpleNamespace + from leapflow.hardware.stream import BatchStreamCoordinator + from leapflow.hardware.alert_policy import AlertRule, HardwareAlertPolicy + base = _context() + context = replace(base, channels=(base.channel("level"), replace(base.channel("level"), channel_id="other"))) + class ReplayIO: + async def read_batch(self, *_: Any) -> Any: + source._stopping.set() + return SimpleNamespace(readings=(_reading(50, sequence=1, at=1), replace(_reading(101, sequence=1, at=1), channel_id="other"))) + observed_guards = [] + class Store: + calls = 0 + def record(self, *_: Any, **__: Any) -> None: + self.calls += 1 + observed_guards.append(policy.fired_count) + raise RuntimeError("persistence is unavailable") + policy = HardwareAlertPolicy([AlertRule(event_kind="threshold_exceeded", action="hw_estop", require_consecutive=1)]) + source = BatchStreamCoordinator(ReplayIO(), context, context.channels, alert_policy=policy, reading_store=Store()) + await asyncio.wait_for(source._run(lambda _: None), timeout=2) + assert policy.fired_count == 1 + assert source._store.calls == 2 + assert observed_guards == [1, 1] diff --git a/tests/test_hw_stop.py b/tests/test_hw_stop.py new file mode 100644 index 00000000..d8edb7a6 --- /dev/null +++ b/tests/test_hw_stop.py @@ -0,0 +1,426 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""Local node stop: OS boundaries, profile isolation, and real CLI/process wiring.""" +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import signal +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from leapflow.cli.commands import hardware +from leapflow.hardware.lhp import node_lifecycle as lifecycle + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX node lifecycle") + + +@pytest.mark.parametrize("argv,expected", [ + (["/env/bin/python3.13", "/env/bin/leap", "hw", "serve", "--node-id", "edge"], True), + (["python", "-u", "-m", "leapflow", "hw", "serve"], True), + (["python", "-m", "leapflow.hardware.lhp", "--node-id=edge"], True), + (["leap", "hw", "stop"], False), + (["leap", "hw", "serve", "--help"], False), + (["sh", "-c", "leap hw serve"], False), + (["python", "fake.py", "leap", "hw", "serve"], False), + (["python", "-c", "print('example')", "-m", "leapflow.hardware.lhp"], False), + (["python", "-c", "from multiprocessing.spawn import spawn_main"], False), + ([], False), +]) +def test_exact_serve_command(argv, expected): + assert lifecycle.is_serve_argv(argv) is expected + + +@pytest.fixture +def world(monkeypatch, tmp_path): + """Fake only process filesystem, systemctl, ps, and signal delivery.""" + runtime = tmp_path / "runtime" + runtime.mkdir() + state = SimpleNamespace(runtime=runtime, processes={}, signals=[], commands=[], + unit_active=True, unit_result="success", unit_pid=123, + accept_stop=True, terminate=True) + read_text, read_bytes = Path.read_text, Path.read_bytes + exists, stat, iterdir = Path.exists, Path.stat, Path.iterdir + + def process(path): + parts = path.parts + if len(parts) > 2 and parts[1] == "proc" and parts[2].isdigit(): + return state.processes.get(int(parts[2])) + return None + + def text(path, *args, **kwargs): + entry = process(path) + if str(path).startswith("/proc/"): + if entry is None or not entry["alive"]: + raise FileNotFoundError(str(path)) + if path.name == "stat": + fields = ["0"] * 20 + fields[0], fields[19] = "S", entry["start"] + return "123 (python) " + " ".join(fields) + if path.name == "cgroup": + return "0::" + entry.get("cgroup", "/session.scope") + return read_text(path, *args, **kwargs) + + def bytes_(path, *args, **kwargs): + entry = process(path) + if entry is not None and entry["alive"]: + if path.name == "cmdline": + return b"\0".join(os.fsencode(arg) for arg in entry["argv"]) + b"\0" + if path.name == "environ": + return b"\0".join(f"{key}={value}".encode() for key, value in entry["env"].items()) + return read_bytes(path, *args, **kwargs) + + def exists_(path): + return True if str(path) in ("/proc/self/stat", "/proc/self/cgroup", "/proc/self") else exists(path) + + def stat_(path, *args, **kwargs): + entry = process(path) + return SimpleNamespace(st_uid=entry.get("uid", os.getuid())) if entry else stat(path, *args, **kwargs) + + def iter_(path): + if str(path) == "/proc": + return iter(Path(f"/proc/{pid}") for pid in state.processes) + return iterdir(path) + + def deliver(pid, signum): + state.signals.append((pid, signum)) + if state.terminate: + affected = {pid} + for child, entry in state.processes.items(): + if entry.get("parent", 0) in affected: + affected.add(child) + for target in affected: + if not state.processes[target].get("retain", False): + state.processes[target]["alive"] = False + + def run(argv, **kwargs): + state.commands.append(argv) + if argv[:2] == ["ps", "-axo"]: + output = "\n".join(f"{pid} {entry.get('parent', 0)}" for pid, entry in state.processes.items() if entry["alive"]) + return subprocess.CompletedProcess(argv, 0, output, "") + if argv[:2] == ["systemctl", "--user"]: + if "stop" in argv: + if not state.accept_stop: + return subprocess.CompletedProcess(argv, 1, "", "refused") + state.unit_active = False + deliver(123, signal.SIGTERM) + return subprocess.CompletedProcess(argv, 0, "", "") + output = f"MainPID={state.unit_pid if state.unit_active else 0}\nActiveState={'active' if state.unit_active else 'inactive'}\nResult={state.unit_result}" + return subprocess.CompletedProcess(argv, 0, output, "") + raise AssertionError(f"unexpected OS call: {argv}") + + monkeypatch.setattr(Path, "read_text", text) + monkeypatch.setattr(Path, "read_bytes", bytes_) + monkeypatch.setattr(Path, "exists", exists_) + monkeypatch.setattr(Path, "stat", stat_) + monkeypatch.setattr(Path, "iterdir", iter_) + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(os, "kill", deliver) + monkeypatch.setattr(os, "pidfd_open", lambda pid, flags: pid, raising=False) + monkeypatch.setattr(signal, "pidfd_send_signal", deliver, raising=False) + monkeypatch.setattr(os, "close", lambda descriptor: None) + + def add(pid=123, node_id="edge", **kwargs): + state.processes[pid] = { + "start": "7", "alive": True, "argv": ["python", "-m", "leapflow", "hw", "serve", "--node-id", node_id], + "env": {"LEAPFLOW_RUNTIME_DIR": str(runtime)}, **kwargs, + } + return lifecycle.NodeProcess(pid=pid, node_id=node_id, start="7") + + state.add = add + return state + + +def test_direct_stop_waits_for_workers(world): + node = world.add() + world.add(pid=124, parent=123) + result = lifecycle.stop_node(node, timeout_s=0.1) + assert result.ok and result.status == "stopped" and not result.survivors + assert world.signals == [(123, signal.SIGTERM)] + assert result.to_dict()["hardware_state_verified"] is False + assert not result.cleanup_complete + assert any("unconfirmed" in line for line in result.consequences) + + +@pytest.mark.parametrize("suffix", ["", "/workers"]) +def test_systemd_stop_does_not_signal_child_directly(world, suffix): + node = world.add(cgroup="/user.slice/user@501.service/app.slice/custom-node.service" + suffix) + result = lifecycle.stop_node(node, timeout_s=0.1) + assert result.ok and result.via == "systemd:custom-node.service" + assert ["systemctl", "--user", "--no-block", "stop", "--", "custom-node.service"] in world.commands + assert not world.unit_active + + +@pytest.mark.parametrize("cause", ["refused", "wrong_main_pid", "system_scope"]) +def test_supervisor_refusal_never_falls_back_to_kill(world, cause): + cgroup = "/system.slice/custom-node.service" if cause == "system_scope" else "/user.slice/user@501.service/app.slice/custom-node.service" + node = world.add(cgroup=cgroup) + if cause == "refused": + world.accept_stop = False + elif cause == "wrong_main_pid": + world.unit_pid = 456 + result = lifecycle.stop_node(node, timeout_s=0.1) + assert not result.ok and result.status == "refused" + assert not world.signals + + +@pytest.mark.parametrize("unit_result", ["timeout", "signal", "core-dump", "exit-code"]) +def test_systemd_abnormal_exit_is_not_reported_as_graceful(world, unit_result): + node = world.add(cgroup="/user.slice/user@501.service/app.slice/custom-node.service") + world.unit_result = unit_result + result = lifecycle.stop_node(node, timeout_s=0.1) + assert not result.ok and result.forced is (unit_result != "exit-code") + assert result.status == "stopped_uncleanly" + assert any("unconfirmed" in line for line in result.consequences) + + +def test_timeout_never_escalates_to_sigkill(world, monkeypatch, capsys): + world.add() + world.terminate = False + monkeypatch.setattr(hardware, "load_config", lambda: SimpleNamespace(runtime_dir=world.runtime)) + args = argparse.Namespace(hw_action="stop", json=True, node_id="edge", timeout=0.01) + assert hardware.cmd_hardware(args) == 1 + result = json.loads(capsys.readouterr().out) + assert not result["ok"] and result["status"] == "timeout" + assert world.signals == [(123, signal.SIGTERM)] + assert result["failure"]["side_effect_state"] == "unknown" + + +def test_changed_identity_and_unrelated_commands_not_signalled(world): + node = world.add(start="8") + assert lifecycle.stop_node(node).status == "not_running" + node = world.add(argv=["python", "other.py"]) + assert lifecycle.stop_node(node).status == "refused" + assert not world.signals + + +def test_discovery_is_profile_and_user_scoped(world): + world.add() + world.add(pid=124, env={"LEAPFLOW_RUNTIME_DIR": str(world.runtime.parent / "other")}) + world.add(pid=125, uid=os.getuid() + 1) + world.add(pid=126, node_id="different") + assert [node.pid for node in lifecycle.discover_nodes(world.runtime, "edge")] == [123] + + +def test_corrupt_stale_and_unrelated_records(world): + (world.runtime / "lhp-node-bad.json").write_text("[]") + stale = world.runtime / "lhp-node-edge-999.json" + stale.write_text(json.dumps({"version": 1, "pid": 999, "node_id": "edge", "start": "7"})) + world.add(argv=["python", "other.py"]) + unrelated = world.runtime / "lhp-node-edge-123.json" + unrelated.write_text(json.dumps({"version": 1, "pid": 123, "node_id": "edge", "start": "7"})) + assert lifecycle.discover_nodes(world.runtime) == [] + assert not stale.exists() and unrelated.exists() + + +def test_cli_selection_and_json_without_initializing_hardware(world, monkeypatch, capsys): + monkeypatch.setattr(hardware, "load_config", lambda: SimpleNamespace(runtime_dir=world.runtime)) + monkeypatch.delenv(lifecycle.NODE_ID_ENV, raising=False) + world.add() + world.add(pid=124, node_id="other") + args = argparse.Namespace(hw_action="stop", json=True, node_id="", timeout=0.1) + assert hardware.cmd_hardware(args) == 1 + assert json.loads(capsys.readouterr().out)["status"] == "ambiguous_node" + assert not world.signals + args.node_id = "edge" + assert hardware.cmd_hardware(args) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["node"]["node_id"] == "edge" + assert payload["consequences"] and payload["failure"] is None + assert world.processes[124]["alive"] + + +def test_human_output_displays_hardware_warnings_without_ansi(world, monkeypatch, capsys): + world.add() + monkeypatch.setattr(hardware, "load_config", lambda: SimpleNamespace(runtime_dir=world.runtime)) + args = argparse.Namespace(hw_action="stop", json=False, node_id="edge", timeout=0.1) + assert hardware.cmd_hardware(args) == 0 + output = capsys.readouterr().out + assert "NOT power-off" in output and "Piper:" in output + assert "NOT verified" in output and "monitoring" in output + assert "\033[" not in output + + +def test_parent_exit_with_worker_survivor_is_not_success(world): + node = world.add() + world.add(pid=124, parent=123, retain=True) + result = lifecycle.stop_node(node, timeout_s=0.01) + assert not result.ok and result.status == "timeout" and result.survivors == (124,) + + +@pytest.mark.parametrize("complete", [True, False]) +def test_cleanup_receipt_is_not_physical_verification(world, complete): + node = world.add() + path = world.runtime / "lhp-node-edge-123.json" + path.write_text(json.dumps({"version": 1, "pid": 123, "start": "7", "node_id": "edge", "cleanup_complete": complete})) + node = lifecycle.NodeProcess(pid=node.pid, node_id=node.node_id, start=node.start, record_path=path) + result = lifecycle.stop_node(node, timeout_s=0.1) + assert result.ok is complete and result.cleanup_complete is complete + assert result.status == ("stopped" if complete else "stopped_uncleanly") + assert not result.to_dict()["hardware_state_verified"] and not path.exists() + + +def test_parent_exit_during_signal_delivery_still_checks_workers(world, monkeypatch): + node = world.add() + world.add(pid=124, parent=123, retain=True) + + def vanished(pid, signum): + world.processes[pid]["alive"] = False + raise ProcessLookupError(pid) + + monkeypatch.setattr(signal, "pidfd_send_signal", vanished) + result = lifecycle.stop_node(node, timeout_s=0.01) + assert not result.ok and result.survivors == (124,) + assert result.status == "timeout" + + +def test_unreadable_supervisor_fails_closed(world, monkeypatch): + node = world.add() + read = Path.read_text + + def unreadable(path, *args, **kwargs): + if str(path) == "/proc/123/cgroup": + raise PermissionError("supervisor unavailable") + return read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", unreadable) + result = lifecycle.stop_node(node, timeout_s=0.01) + assert not result.ok and result.status == "refused" + assert not world.signals + + +def test_worker_pid_reuse_is_not_counted_as_a_survivor(world, monkeypatch): + node = world.add() + world.add(pid=124, parent=123) + + def exit_and_recycle(pid, signum): + world.signals.append((pid, signum)) + world.processes[pid]["alive"] = False + world.processes[124]["start"] = "8" + + monkeypatch.setattr(signal, "pidfd_send_signal", exit_and_recycle) + result = lifecycle.stop_node(node, timeout_s=0.01) + assert result.ok and not result.survivors + assert world.signals == [(123, signal.SIGTERM)] + + +@pytest.mark.component +def test_records_are_private_atomic_and_ignore_predictable_temp_symlinks(tmp_path): + victim = tmp_path / "unrelated" + victim.write_text("preserve") + legacy_temp = tmp_path / f"lhp-node-edge-{os.getpid()}.tmp" + legacy_temp.symlink_to(victim) + path = lifecycle.write_record(tmp_path, node_id="edge", endpoint="tcp://127.0.0.1:1234") + assert path is not None + assert path.stat().st_mode & 0o777 == 0o600 + assert json.loads(path.read_text())["cleanup_complete"] is False + lifecycle.finish_record(path, complete=True) + assert json.loads(path.read_text())["cleanup_complete"] is True + assert path.stat().st_mode & 0o777 == 0o600 + assert victim.read_text() == "preserve" + + +@pytest.mark.parametrize("timeout", [-1, 0, float("nan"), float("inf")]) +def test_invalid_timeout_has_no_side_effects(world, capsys, timeout): + args = argparse.Namespace(hw_action="stop", json=True, node_id="", timeout=timeout) + assert hardware.cmd_hardware(args) == 1 + assert json.loads(capsys.readouterr().out)["status"] == "invalid_args" + assert not world.commands and not world.signals + + +def test_cli_not_running_is_idempotent(world, monkeypatch, capsys): + monkeypatch.setattr(hardware, "load_config", lambda: SimpleNamespace(runtime_dir=world.runtime)) + assert hardware.cmd_hardware(argparse.Namespace(hw_action="stop", json=True, node_id="", timeout=1)) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "not_running" + assert not world.signals + + +@pytest.mark.component +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["server", "skills"]) +async def test_serve_shutdown_failure_still_closes_remaining_owners(tmp_path, monkeypatch, failure): + from leapflow.hardware.lhp.runtime import serve_node + from leapflow.hardware.lhp.server import LhpNodeServer + from leapflow.hardware.registry import HardwareRegistry + from leapflow.hardware.skills.runtime import SkillRuntime + + for key, value in { + "LEAPFLOW_DATA_DIR": str(tmp_path), "LEAPFLOW_PROFILE": "cleanup_test", + "LEAPFLOW_RUNTIME_DIR": str(tmp_path / "runtime"), + "LEAPFLOW_HARDWARE_PROVIDERS": "yaml", "LEAPFLOW_HARDWARE_DEVICES_DIR": str(tmp_path / "devices"), + "LEAPFLOW_WORKSPACE_ROOT": str(tmp_path), + }.items(): + monkeypatch.setenv(key, value) + callbacks, closed = [], [] + monkeypatch.setattr(asyncio.get_running_loop(), "add_signal_handler", + lambda sig, callback: callbacks.append(callback)) + start = LhpNodeServer.start + + async def start_then_stop(self, **kwargs): + result = await start(self, **kwargs) + callbacks[0]() + return result + + monkeypatch.setattr(LhpNodeServer, "start", start_then_stop) + + def track_close(name, closer): + async def close(self): + await closer(self) + closed.append(name) + if name == failure: + raise OSError("injected cleanup failure") + return close + + monkeypatch.setattr(LhpNodeServer, "stop", track_close("server", LhpNodeServer.stop)) + monkeypatch.setattr(SkillRuntime, "close", track_close("skills", SkillRuntime.close)) + monkeypatch.setattr(HardwareRegistry, "close_all", track_close("registry", HardwareRegistry.close_all)) + with pytest.raises(OSError, match="injected cleanup failure"): + await serve_node(node_id="cleanup_test", port=0) + assert closed == ["server", "skills", "registry"] + records = list((tmp_path / "runtime").glob("lhp-node-*.json")) + assert len(records) == 1 + assert json.loads(records[0].read_text())["cleanup_complete"] is False + + +@pytest.mark.parametrize("entry", [("leapflow", "hw", "serve"), ("leapflow.hardware.lhp",)]) +@pytest.mark.component +@pytest.mark.asyncio +async def test_real_cli_serve_then_stop_and_repeat(tmp_path, entry): + """No mocked wiring: actual empty node, profile records, CLI parser and SIGTERM.""" + env = {**os.environ, "LEAPFLOW_DATA_DIR": str(tmp_path), "LEAPFLOW_PROFILE": "stop_test", + "LEAPFLOW_RUNTIME_DIR": str(tmp_path / "runtime"), "LEAPFLOW_FLEET_NODES": "[]", + "LEAPFLOW_HARDWARE_PROVIDERS": "yaml", "LEAPFLOW_HARDWARE_DEVICES_DIR": str(tmp_path / "devices"), + "LEAPFLOW_WORKSPACE_ROOT": str(tmp_path)} + for key in ("LEAPFLOW_LHP_NODE_ID", "LEAPFLOW_LHP_TOKEN", "LEAPFLOW_LHP_PERMIT_ISSUER_KEYS"): + env.pop(key, None) + server = await asyncio.create_subprocess_exec( + sys.executable, "-m", *entry, "--node-id", "stop_test", "--port", "0", + env=env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + ) + try: + line = await asyncio.wait_for(server.stdout.readline(), 30) + assert line, (await server.stderr.read()).decode() + ready = json.loads(line) + assert ready["event"] == "lhp_node_ready" and ready["device_count"] == 0 + for expected in ("stopped", "not_running"): + command = await asyncio.create_subprocess_exec( + sys.executable, "-m", "leapflow", "hw", "stop", "--node-id", "stop_test", "--json", + env=env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(command.communicate(), 30) + assert command.returncode == 0, stderr.decode() + result = json.loads(stdout) + assert result["status"] == expected + if expected == "stopped": + assert result["cleanup_complete"] and not result["hardware_state_verified"] + assert await asyncio.wait_for(server.wait(), 5) == 0 + assert not list((tmp_path / "runtime").glob("lhp-node-*.json")) + finally: + if server.returncode is None: + server.terminate() + await asyncio.wait_for(server.wait(), 10) diff --git a/tests/test_leapcalib_generations.py b/tests/test_leapcalib_generations.py index 922316f0..02bef6e9 100644 --- a/tests/test_leapcalib_generations.py +++ b/tests/test_leapcalib_generations.py @@ -17,7 +17,7 @@ from leapflow.hardware.calibration_store import CalibrationRecord, CalibrationStore from leapflow.hardware.lhp.control_sessions import NodeSessions from leapflow.hardware.node_gate import NodeWriteGate -from leapflow.hardware.transport import WriteOutcome +from leapflow.hardware.transport import TransportStatus, WriteOutcome from leapflow.storage.connection import LocalConnectionHolder from tests._harness.hardware_stubs import StubNodeRegistry @@ -199,6 +199,122 @@ def real_registry(tmp_path: Path) -> HardwareRegistry: return registry +@pytest.mark.parametrize("verified", [True, False, None]) +@pytest.mark.parametrize("batch", [False, True]) +async def test_registry_halt_settles_only_verified_physical_uncertainty( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + verified: bool | None, batch: bool, +) -> None: + registry = real_registry(tmp_path) + sessions, device = registry.node_sessions, "conformance_device" + try: + transport = await registry.transport(device) + original_write = transport.write + + async def uncertain(channel_id: str, value: object) -> WriteOutcome: + return WriteOutcome(ok=False, side_effect_state="unknown", error="target receipt lost") + + async def halt() -> TransportStatus: + return TransportStatus(connected=True, stop_verified=verified, detail="test halt evidence") + + monkeypatch.setattr(transport, "write", uncertain) + monkeypatch.setattr(transport, "halt", halt) + gate = NodeWriteGate(registry) + if batch: + ok, outcomes = await gate.write_batch(device, (("setpoint", 1),)) + assert not ok and outcomes[0].side_effect_state == "unknown" + else: + assert (await gate.write(device, "setpoint", 1)).side_effect_state == "unknown" + assert sessions.busy_device((device,)) == device + refusal = await gate.write(device, "setpoint", 2) + assert not refusal.ok and "target receipt lost" in refusal.error + assert "hw_estop" in refusal.error + assert "unsettled dispatch effect" in caplog.text + status = await registry.halt(device) + assert status.stop_verified is verified + monkeypatch.setattr(transport, "write", original_write) + if verified: + assert sessions.busy_device((device,)) == "" + assert (await gate.write(device, "setpoint", 2)).ok + assert "settled physical uncertainty after verified halt" in caplog.text + else: + assert sessions.busy_device((device,)) == device + assert not (await gate.write(device, "setpoint", 2)).ok + finally: + await registry.close_all() + + +async def test_halt_cannot_clear_a_failure_recorded_while_it_is_in_flight( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = real_registry(tmp_path) + sessions, device = registry.node_sessions, "conformance_device" + entered, release = asyncio.Event(), asyncio.Event() + stopping = None + try: + transport = await registry.transport(device) + first = sessions.begin_dispatch(device) + second = sessions.begin_dispatch(device) + sessions.finish_dispatch(first, settled=False, detail="old failure") + + async def halt() -> TransportStatus: + entered.set() + await release.wait() + return TransportStatus(connected=True, stop_verified=True) + + monkeypatch.setattr(transport, "halt", halt) + stopping = asyncio.create_task(registry.halt(device)) + await entered.wait() + sessions.finish_dispatch(second, settled=False, detail="new failure during halt") + release.set() + assert (await stopping).stop_verified is True + assert sessions.busy_device((device,)) == device + assert "new failure during halt" in sessions.busy_message(device) + assert (await registry.halt(device)).stop_verified is True + assert sessions.busy_device((device,)) == "" + finally: + release.set() + if stopping is not None: + await stopping + await registry.close_all() + + +@pytest.mark.parametrize("error_type", [OSError, asyncio.CancelledError]) +async def test_failed_or_cancelled_halt_retains_uncertainty( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error_type: type[BaseException], +) -> None: + registry = real_registry(tmp_path) + sessions, device = registry.node_sessions, "conformance_device" + transport = await registry.transport(device) + original_halt = transport.halt + + async def halt() -> TransportStatus: + raise error_type("halt interrupted") + + try: + token = sessions.begin_dispatch(device) + sessions.finish_dispatch(token, settled=False) + monkeypatch.setattr(transport, "halt", halt) + with pytest.raises(error_type): + await registry.halt(device) + assert sessions.busy_device((device,)) == device + finally: + monkeypatch.setattr(transport, "halt", original_halt) + await registry.close_all() + + +async def test_missing_link_does_not_clear_uncertainty(tmp_path: Path) -> None: + registry = real_registry(tmp_path) + sessions, device = registry.node_sessions, "conformance_device" + try: + token = sessions.begin_dispatch(device) + sessions.finish_dispatch(token, settled=False) + assert await registry.halt(device) is None + assert sessions.busy_device((device,)) == device + finally: + await registry.close_all() + + async def test_real_registry_lazy_shared_holder_and_private_cache_binding(tmp_path: Path) -> None: registry = real_registry(tmp_path) try: @@ -391,6 +507,8 @@ async def test_reserved_failure_preserves_generation_and_releases_only_after_set assert not sessions.reserved_device("conformance_device") assert await asyncio.to_thread(store.active, "cell") == first assert await asyncio.to_thread(store.committed_request, "cell", "failure") is None + assert not sessions.settle_halt("conformance_device", before=float("inf"), evidence="verified") + assert "unsettled transaction effect" in sessions.busy_message("conformance_device") await sessions.shutdown() with pytest.raises(ValueError, match="busy"): await sessions.run_reserved( @@ -447,6 +565,9 @@ def broken(generation): assert not sessions.reserved_device("conformance_device") assert sessions.unsettled_device("conformance_device") assert sessions.unsettled_device("consumer:precision") + assert not sessions.settle_halt("conformance_device", before=float("inf"), evidence="verified") + assert not sessions.settle_halt("consumer:precision", before=float("inf"), evidence="verified") + assert "a halt cannot settle a model transaction" in sessions.busy_message("conformance_device") with pytest.raises(ValueError, match="unsettled"): sessions.reserve(("consumer:precision",), owner_id="other") assert not (await NodeWriteGate(registry).write("conformance_device", "setpoint", 1)).ok @@ -584,8 +705,8 @@ def test_pointer_journal_uses_commit_order_not_wall_clock(tmp_path, monkeypatch) store = GenerationStore(holder) clock = iter([900, 100, 100, 50, 1]) monkeypatch.setattr(generations.time, "time", lambda: next(clock)) - first = commit(store, "one") - commit(store, "two", 1) + first = commit(store, "one", model={"device_id": "test_arm", "synthetic": True, "value": "one"}) + commit(store, "two", 1, model={"device_id": "test_arm", "synthetic": True, "value": "two"}) assert ( store.rollback( workcell_id="cell", @@ -605,6 +726,15 @@ def test_pointer_journal_uses_commit_order_not_wall_clock(tmp_path, monkeypatch) causal={}, ) assert GenerationStore(holder).invalidation("cell")["request_id"] == "invalid" + expected = ["one", "two", "back", "invalid"] + assert [event["request_id"] for event in store.pending_events()] == expected + assert [event["request_id"] for event in store.pending_events(device_id="test_arm")] == expected + delivered = [] + while page := store.pending_events(limit=2, device_id="test_arm"): + delivered.extend(event["request_id"] for event in page) + for event in page: + store.acknowledge_event(event["event_id"]) + assert delivered == expected # pagination cannot replay a prior scope after invalidation finally: holder.close() diff --git a/tests/test_leapcalib_readiness.py b/tests/test_leapcalib_readiness.py index 0d4544a4..98e6af08 100644 --- a/tests/test_leapcalib_readiness.py +++ b/tests/test_leapcalib_readiness.py @@ -353,6 +353,7 @@ async def test_actual_node_startup_binds_evidence_cache_or_refuses_diagnostics( ) config = SimpleNamespace( profile_layout=layout, + runtime_dir=layout.runtime_dir, profile_manifest=SimpleNamespace(profile_id="test_profile"), hardware_providers="yaml", hardware_enabled=True, diff --git a/tests/test_lhp_control_lease.py b/tests/test_lhp_control_lease.py index f4d74114..6bcf4717 100644 --- a/tests/test_lhp_control_lease.py +++ b/tests/test_lhp_control_lease.py @@ -9,6 +9,7 @@ from __future__ import annotations import asyncio +import time from dataclasses import dataclass, field from types import SimpleNamespace from typing import Any @@ -412,6 +413,28 @@ def discover(self): registry.start_control_loop("remote", object()) +@pytest.mark.parametrize("kind", ["dispatch", "control"]) +def test_verified_halt_settlement_is_device_scoped_and_keeps_active_claims(kind: str) -> None: + sessions, _ = _make_sessions() + if kind == "control": + token = sessions.begin_control("arm", stop=lambda: None) + sessions.finish_control(token, settled=False) + else: + token = sessions.begin_dispatch("arm") + sessions.finish_dispatch(token, settled=False) + other = sessions.begin_dispatch("other") + sessions.finish_dispatch(other, settled=False) + active = sessions.begin_dispatch("active") + before = time.monotonic() + assert sessions.busy_device(("arm",)) == "arm" + assert sessions.settle_halt("arm", before=before, evidence="verified hold") + assert sessions.busy_device(("arm",)) == "" + assert sessions.busy_device(("other",)) == "other" + assert not sessions.settle_halt("active", before=before, evidence="verified hold") + assert sessions.busy_device(("active",)) == "active" + sessions.finish_dispatch(active, settled=True) + + # ── Shutdown ───────────────────────────────────────────────────────────── diff --git a/tests/test_lhp_fabric.py b/tests/test_lhp_fabric.py index ca1cf966..ebef4d08 100644 --- a/tests/test_lhp_fabric.py +++ b/tests/test_lhp_fabric.py @@ -2048,3 +2048,28 @@ def _with_transport(context: HardwareContext, transport: TransportRef) -> Hardwa from dataclasses import replace return replace(context, transport=transport) + + +def test_remote_physical_rate_uses_source_intervals_not_network_arrival() -> None: + from dataclasses import replace + from leapflow.hardware.context import Channel, Envelope + from leapflow.hardware.stream import HardwareEventDetector, EventKind, ReadingRing + from leapflow.hardware.transports.lhp import LhpTransport + transport = LhpTransport({"remote_device_id": "arm"}) + channel = Channel(channel_id="angle", direction="read", quantity="angle", unit="deg", sample_rate_hz=10, + envelope=Envelope(declared=True, min_value=-180, max_value=180, max_rate=50)) + detector = HardwareEventDetector(_node_device("arm"), channel) + ring = ReadingRing() + for index, value in enumerate([179, -179, -177]): + reading = transport._localize_reading({"channel_id": "angle", "value": value, + "observed_at": 1_787_000_000 + index * .1, "sequence": 1 + index * 20, + "sequence_scope": "device_read", "acquisition_at": 5000 + index * .1, + "acquisition_epoch": "node-boot"}, "angle") + assert reading.acquisition_at == 5000 + index * .1 + assert reading.rate_timebase == "source" + reading = replace(reading, monotonic_at=1 + index * .00001) + assert ring.record(reading) == 0 + assert EventKind.RATE_EXCEEDED not in [e.kind for e in detector.observe(reading)] + old_peer = transport._localize_reading({"channel_id": "angle", "value": 0, "sequence": 100}, "angle") + assert old_peer.sequence_scope == "unknown" and old_peer.rate_timebase == "unavailable" + assert EventKind.RATE_EXCEEDED not in [e.kind for e in detector.observe(old_peer)] diff --git a/tests/test_lhp_skill_methods.py b/tests/test_lhp_skill_methods.py index ff18c78f..ffba6bb2 100644 --- a/tests/test_lhp_skill_methods.py +++ b/tests/test_lhp_skill_methods.py @@ -252,6 +252,21 @@ async def test_busy_refusal_does_not_burn_the_permit(tmp_path: Path) -> None: assert "run" in retried, "the same permit still authorizes the plan once the device is free" await node.until_done(retried["run"]["run_id"]) + third = await node.plan(SEQUENCE, move_by(10)) + request = {"plan_digest": third["plan"]["digest"], "permit": node.permit(third["permit_digest"])} + token = node.server.sessions.begin_dispatch(ARM_ID) + node.server.sessions.finish_dispatch(token, settled=False, detail="target receipt lost") + blocked = await node.call("skill.execute", request) + assert blocked["outcome"]["failure_code"] == FAILURE_DEVICE_BUSY + assert "target receipt lost" in blocked["outcome"]["message"] + assert "hw_estop" in blocked["outcome"]["message"] + written = await node.call("write", {"device_id": ARM_ID, "channel_id": "joint.j1.position", "value": 0}) + assert written["failure_code"] == FAILURE_DEVICE_BUSY + assert "target receipt lost" in written["error"] and "hw_estop" in written["error"] + assert node.server._authority.verifier.verify( + ExecutionPermit.from_dict(request["permit"]), expected_digest=third["permit_digest"], + ) is None, "uncertainty must refuse before consuming the permit" + async def test_generation_stale_refusal_preserves_exact_permit(tmp_path: Path) -> None: node = _Node(tmp_path, write_mode=WRITE_MODE_PERMIT_REQUIRED) diff --git a/tests/test_piper_transport.py b/tests/test_piper_transport.py index eb5f8fd8..74e01d83 100644 --- a/tests/test_piper_transport.py +++ b/tests/test_piper_transport.py @@ -841,6 +841,69 @@ def test_a_write_still_refuses_a_ledger_that_never_coheres(monkeypatch: pytest.M assert result.side_effect_state == SIDE_EFFECT_NONE and arm.events == before +@pytest.mark.parametrize("phase", ["hold_read", "mode_selection"]) +@pytest.mark.parametrize("recovers", [True, False]) +def test_inner_motion_gate_requalifies_or_refuses_before_target( + monkeypatch: pytest.MonkeyPatch, phase: str, recovers: bool, +) -> None: + """Skew after admission is boundedly waited for and never called an uncertain send.""" + transport, arm, _ = _open_transport(monkeypatch, {"receive_max_skew_s": .02, + "receive_requalify_s": .2}) + timers: list[threading.Timer] = [] + + def skew() -> None: + _skewed(arm, .04) + if recovers: + timer = threading.Timer(.03, arm.receive) + timers.append(timer) + timer.start() + + if phase == "hold_read": + original_read = arm.get_joint_angles + injected = False + + def read() -> Any: + nonlocal injected + if not injected: + injected = True + skew() + return original_read() + + monkeypatch.setattr(arm, "get_joint_angles", read) + else: + original_mode = arm.set_motion_mode + + def mode(name: str) -> None: + original_mode(name) + skew() + + monkeypatch.setattr(arm, "set_motion_mode", mode) + try: + result = _write(transport, "joint.j1.position", 15) + if recovers: + assert result.ok and result.side_effect_state == SIDE_EFFECT_COMMITTED + assert len(arm.joint_calls) == 1 + else: + assert not result.ok and result.side_effect_state == SIDE_EFFECT_NONE + assert result.failure_code == "piper_unqualified_feedback" + assert "no motion target was sent" in result.error + assert not arm.joint_calls + finally: + for timer in timers: + timer.join() + + +def test_motion_can_mode_refusal_is_not_an_uncertain_target( + monkeypatch: pytest.MonkeyPatch, +) -> None: + transport, arm, _ = _open_transport(monkeypatch) + arm.ignore_mode = True + result = _write(transport, "joint.j1.position", 15) + assert not result.ok and result.side_effect_state == SIDE_EFFECT_NONE + assert result.failure_code == "piper_mode_switch_refused" + assert not arm.joint_calls + + def test_an_enable_waits_for_a_skewed_ledger_too(monkeypatch: pytest.MonkeyPatch) -> None: transport, arm, _ = _open_transport(monkeypatch, {"receive_max_skew_s": .05, "receive_requalify_s": .5}) @@ -884,10 +947,21 @@ def lose(speed: int) -> None: arm.receive(dlc=0) monkeypatch.setattr(arm, "set_speed_percent", lose) result = _write(transport, channel, value) - assert not result.ok and result.side_effect_state in (SIDE_EFFECT_COMMITTED, SIDE_EFFECT_UNKNOWN) + expected = SIDE_EFFECT_NONE if channel == "ee.x" else SIDE_EFFECT_COMMITTED + assert not result.ok and result.side_effect_state == expected assert not arm.pose_calls and not arm.gripper.calls +def test_angle_mode_does_not_convert_stroke_or_suppress_signed_force(monkeypatch: pytest.MonkeyPatch) -> None: + transport, arm, _ = _open_transport(monkeypatch) + monkeypatch.setattr(arm.gripper, "get_gripper_status", lambda: _message( + SimpleNamespace(value=35.0, mode="angle", force=-0.7))) + with pytest.raises(TransportError, match="unavailable"): + asyncio.run(transport.read("gripper.position")) + assert asyncio.run(transport.read("gripper.effort")).value == -0.7 + assert not arm.gripper.calls + + def test_gripper_stroke_force_pair_and_mode_guard(monkeypatch: pytest.MonkeyPatch) -> None: transport, arm, _ = _open_transport(monkeypatch) assert _write(transport, "gripper.effort", 2).ok @@ -952,7 +1026,7 @@ def test_a_firmware_that_never_latches_the_mode_sends_no_target( arm.ignore_move_mode = True arm.mode_feedback = 0xFF result = _write(transport, "joint.j1.position", 12.5) - assert not result.ok and result.side_effect_state == SIDE_EFFECT_UNKNOWN + assert not result.ok and result.side_effect_state == SIDE_EFFECT_NONE assert result.failure_code == "piper_move_mode_refused" assert "unknown(0xFF)" in (result.error or "") and "no target was sent" in (result.error or "") assert not arm.joint_calls