From ef16dde9fd17ad92de69ec5e31efc7e90ca7cf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 12:42:26 +0200 Subject: [PATCH 1/5] docs(daemon-client): document daemon-drain-timeout and the post-drain force retire Retire through the typed result instead of casting the thrown error's details. Co-Authored-By: Claude Sonnet 5.5 --- src/commands/schema/cli-help.ts | 2 +- .../daemon-client-ensure-drain.test.ts | 1 + src/daemon-client/daemon-client-ensure.ts | 38 +++++++------------ src/daemon-client/daemon-client-lifecycle.ts | 34 +++++++++++------ website/docs/docs/client-api.md | 3 +- 5 files changed, 41 insertions(+), 37 deletions(-) diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 4629938f10..13cf984015 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -508,7 +508,7 @@ Runner and daemon lifecycle (applies to simulators too): No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. - A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. + A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts b/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts index e64f1ea987..d4d21125a7 100644 --- a/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts +++ b/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts @@ -194,6 +194,7 @@ test('a daemon still running past the deadline that answers no probe is force-re } }); +// A drain that never finishes while the daemon keeps serving: it accepted the request but is stuck. test('a daemon still running past the deadline that answers a probe is left alone', async (t) => { if (!(await withLoopback(t))) return; const { stateDir, paths } = stateDirAndPaths(); diff --git a/src/daemon-client/daemon-client-ensure.ts b/src/daemon-client/daemon-client-ensure.ts index 7cdb50df4a..3016b1b9ec 100644 --- a/src/daemon-client/daemon-client-ensure.ts +++ b/src/daemon-client/daemon-client-ensure.ts @@ -4,17 +4,16 @@ import type { EnsureDaemonOptions, } from '@agent-device/contracts/client'; import { waitForDaemonExit } from '@agent-device/daemon-contracts/daemon-process'; -import { - recoverAbandonedDaemonRegistration, - type DaemonRetirementResult, -} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { recoverAbandonedDaemonRegistration } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { sleep } from '@agent-device/host-kit/retry'; import { AppError } from '@agent-device/kernel/errors'; import { assertDaemonPolicyMatches, observeLocalDaemon, resolveLocalDaemonSettings, + daemonRetirementUnconfirmedError, retireDaemonForTakeover, + stopDaemonForTakeover, startLocalDaemon, type DaemonClientSettings, type EnsuredDaemon, @@ -164,27 +163,18 @@ async function awaitDrainedDaemon( return; } if (await answersLivenessProbe(info)) throw await daemonDrainTimeoutError(info, settings); - try { - await retireDaemonForTakeover(info, settings.paths, 'force'); - } catch (error) { - if (isRefusalToSignalNonDaemon(error)) throw await daemonDrainTimeoutError(info, settings); - throw error; - } -} - -/** - * The identity check could not prove the live process is the daemon it observed (an embedded host - * runs no daemon entry; a start time or command line may also be unreadable), so nothing was signaled. - */ -function isRefusalToSignalNonDaemon(error: unknown): boolean { - if (!(error instanceof AppError) || error.details?.reason !== 'daemon_retirement_unconfirmed') { - return false; + const retirement = await stopDaemonForTakeover(info, settings.paths, 'force'); + if (retirement.status !== 'retained') return; + // The identity check could not prove the live process is the daemon it observed (an embedded host + // runs no daemon entry; a start time or command line may also be unreadable), so nothing was + // signaled. + if ( + retirement.termination?.status === 'retained' && + retirement.termination.reason === 'identity-unverified' + ) { + throw await daemonDrainTimeoutError(info, settings); } - const { termination } = error.details.retirement as Extract< - DaemonRetirementResult, - { status: 'retained' } - >; - return termination?.status === 'retained' && termination.reason === 'identity-unverified'; + throw daemonRetirementUnconfirmedError(retirement, settings.paths); } /** diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 536cb052e4..be2869c996 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -354,24 +354,36 @@ function daemonRegistrationUnprovenError( }); } -export async function retireDaemonForTakeover( +export async function stopDaemonForTakeover( existing: DaemonInfo, paths: DaemonPaths, mode: 'graceful' | 'force' = 'graceful', -): Promise { - const retirement = await stopAndRetireDaemon({ +): Promise { + return await stopAndRetireDaemon({ paths: paths, observed: { pid: existing.pid, startTime: existing.processStartTime ?? null }, mode, }); - if (retirement.status === 'retained') { - throw new AppError('COMMAND_FAILED', 'Daemon replacement could not be confirmed.', { - reason: 'daemon_retirement_unconfirmed', - retirement, - hint: - retirement.error?.hint ?? resolveDaemonStartupHint(getDaemonMetadataState(paths), paths), - }); - } +} + +export function daemonRetirementUnconfirmedError( + retirement: Extract, + paths: DaemonPaths, +): AppError { + return new AppError('COMMAND_FAILED', 'Daemon replacement could not be confirmed.', { + reason: 'daemon_retirement_unconfirmed', + retirement, + hint: retirement.error?.hint ?? resolveDaemonStartupHint(getDaemonMetadataState(paths), paths), + }); +} + +export async function retireDaemonForTakeover( + existing: DaemonInfo, + paths: DaemonPaths, + mode: 'graceful' | 'force' = 'graceful', +): Promise { + const retirement = await stopDaemonForTakeover(existing, paths, mode); + if (retirement.status === 'retained') throw daemonRetirementUnconfirmedError(retirement, paths); } /** diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 7400b7a5e3..c0712f2c54 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -109,7 +109,7 @@ await ensureDaemon({ const client = createAgentDeviceClient({ stateDir, daemonLifecycle: 'attach' }); ``` -`ensureDaemon()` starts a daemon when none runs. It replaces a daemon that does not match the client only if that daemon holds no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise it fails and leaves the daemon alone. Two cases retire a daemon without that proof, because it cannot be asked: a registered daemon that answers nothing on any transport, and one that times out on the quiesce request and then fails repeated liveness probes. It returns `{ stateDir, daemon, started }`; `daemon` holds the pid, version and code identity, and no credentials. `onStart` runs when the call started a daemon: `'incompatible'` when it replaced a daemon that agreed to leave, with that daemon as `previous`, and `'missing'` otherwise. A `'missing'` start that also has a `previous` replaced a registered daemon that answered nothing. +`ensureDaemon()` starts a daemon when none runs. It replaces a daemon that does not match the client only if that daemon holds no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise it fails and leaves the daemon alone. Two cases retire a daemon without that proof, because it cannot be asked: a registered daemon that answers nothing on any transport, and one that times out on the quiesce request and then fails repeated liveness probes. A daemon that accepted the quiesce request is never signaled while it drains: the call waits up to 60 s for it to exit and clear its registration. One still running after that fails repeated liveness probes and is force-retired when its process is provably a daemon; otherwise the call fails with `daemon-drain-timeout`. It returns `{ stateDir, daemon, started }`; `daemon` holds the pid, version and code identity, and no credentials. `onStart` runs when the call started a daemon: `'incompatible'` when it replaced a daemon that agreed to leave, with that daemon as `previous`, and `'missing'` otherwise. A `'missing'` start that also has a `previous` replaced a registered daemon that answered nothing. These failures carry a typed `error.details.reason`: @@ -118,6 +118,7 @@ These failures carry a typed `error.details.reason`: | `daemon-not-running` | An `'attach'` client found no daemon to connect to. Call `ensureDaemon()`. | | `daemon-incompatible` | The running daemon does not match the client. `details.daemon` and `details.client` hold both identities. Stop it with `agent-device daemon stop --state-dir `, then call `ensureDaemon()`. A daemon from a release that predates quiescing cannot be asked to leave, so `ensureDaemon()` reports it the same way. | | `daemon-busy` | `ensureDaemon()` found an incompatible daemon that still holds work. `details` counts `inFlightRequests`, `sessions`, `repairSessions` and `leases`. Wait for them to end and retry. | +| `daemon-drain-timeout` | A daemon accepted the request to leave but was still running 60 s later and could not be force-retired, either because it still answers or because its process cannot be proven to be a daemon (an embedded host). It was not signaled. Retry once it has exited, or stop it with `agent-device daemon stop --state-dir `. | `ensureDaemon()` only ever manages the local daemon on `stateDir`. A client with `daemonBaseUrl` never starts or stops a local daemon in either mode, and `daemonLifecycle` does not change `daemonTransport` or `daemonServerMode`. From 0da0a66886533193ea826cb5d76e960d6eaff65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 12:45:22 +0200 Subject: [PATCH 2/5] fix(daemon-client): name the state dir in the not-running hint and report a force-retired drain honestly Part of #3747, refs #3754 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 1 + packages/daemon/src/core/session-store.ts | 3 +- .../daemon-client-ensure-drain.test.ts | 9 ++++- .../daemon-client-explicit-lifecycle.test.ts | 38 ++++++++++++++++++- .../__tests__/explicit-lifecycle-fixture.ts | 2 +- src/daemon-client/daemon-client-ensure.ts | 11 +++--- src/daemon-client/daemon-client-ensuring.ts | 5 +-- src/daemon-client/daemon-lifecycle-errors.ts | 11 +++++- 8 files changed, 67 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9c8b4468..5e7d8d8b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- The `daemon-not-running` hint now names `--state-dir ` when the state directory is not the default, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`. - Fix `ensureDaemon()`, the CLI and MCP killing a daemon they had just asked to leave. After the daemon answers the quiesce request with `draining`, the client now waits up to 60 s for it to exit by itself and clean up its registration, so plugin disposal and lease finalization finish (on Windows a signal ends the process outright, and on POSIX a drain longer than 3 s was killed). A daemon still running at the deadline that fails its liveness probes is force-retired; one that still answers is left running and the call fails with `details.reason: 'daemon-drain-timeout'`, with a hint naming `agent-device daemon stop --state-dir `. - `@agent-device/testmu` is now a version 2 provider plugin. `import testmu from '@agent-device/testmu'` gives a factory, `testmu({ credentials: { username, accessKey } })`, that reads those credentials instead of `LT_USERNAME` and `LT_ACCESS_KEY`; without options it reads the variables as before. The package still declares the version 1 plugin entry, so `agent-device plugins add @agent-device/testmu` keeps working on hosts that predate the version 2 contract. - Provider plugins have a new contract. A package exports a factory at `./agent-device` that returns `{ apiVersion: 2, provider, credentials, connection?, setup(host) }`, typed by `AgentDevicePluginFactory` from `agent-device/plugins`. The store calls the factory with the package's `config.json` options, and a JavaScript host can call it directly. Version 1 plugins (`agentDevicePlugin.apiVersion: 1`) keep loading. diff --git a/packages/daemon/src/core/session-store.ts b/packages/daemon/src/core/session-store.ts index d8e752acde..ccd50d1a37 100644 --- a/packages/daemon/src/core/session-store.ts +++ b/packages/daemon/src/core/session-store.ts @@ -3,6 +3,7 @@ import { resolveSessionAppLogPath, resolveSessionAppLogPidPath, } from '@agent-device/daemon-contracts/session-artifact-paths'; +import { DAEMON_SHUTTING_DOWN_REASON } from '@agent-device/daemon-contracts/daemon-resolution'; import path from 'node:path'; import fs from 'node:fs'; import { AppError } from '@agent-device/kernel/errors'; @@ -74,7 +75,7 @@ export class SessionStore { assertAdmissionOpen(address: string): void { if (!this.acceptingSessions) { throw new AppError('COMMAND_FAILED', 'Daemon is shutting down', { - reason: 'daemon_shutting_down', + reason: DAEMON_SHUTTING_DOWN_REASON, session: address, }); } diff --git a/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts b/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts index d4d21125a7..91c5da1311 100644 --- a/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts +++ b/src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts @@ -179,9 +179,16 @@ test('a daemon still running past the deadline that answers no probe is force-re launchesFreshDaemonAt(spawn, paths, fresh.port); try { - const connection = await ensureDaemon({ stateDir, daemonTransport: 'http' }); + const started: Array<[string, number | undefined]> = []; + const connection = await ensureDaemon({ + stateDir, + daemonTransport: 'http', + onStart: (reason, previous) => started.push([reason, previous?.pid]), + }); assert.equal(connection.started, true); + // Force-retired for outlasting its drain, it was never shown idle: reported as a missing daemon. + assert.deepEqual(started, [['missing', childPid]]); assert.equal(drain.signals.length > 0, true, 'the stuck daemon was signaled'); assert.ok( drain.signals.every(({ atMs }) => atMs >= DRAIN_DEADLINE_MS), diff --git a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts index 0655501864..fc050514bf 100644 --- a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts @@ -4,7 +4,10 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { closeLoopbackServer, supportsLoopbackBind } from '@agent-device/test-kit'; import { AppError } from '@agent-device/kernel/errors'; -import { DAEMON_QUIESCE_COMMAND } from '@agent-device/daemon-contracts/daemon-resolution'; +import { + DAEMON_QUIESCE_COMMAND, + resolveDaemonPaths, +} from '@agent-device/daemon-contracts/daemon-resolution'; import { DAEMON_STARTUP_EXIT_CODES } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { isProcessAlive } from '@agent-device/host-kit/process'; import { readVersion } from '@agent-device/host-kit/version'; @@ -39,6 +42,7 @@ import { sleep } from '@agent-device/host-kit/retry'; import * as transport from '../daemon-client-transport.ts'; import { sendToDaemon } from '../daemon-client.ts'; import { ensureDaemon } from '../daemon-client-ensure.ts'; +import { daemonNotRunningError } from '../daemon-lifecycle-errors.ts'; // The explicit lifecycle: what an attach client may never do, and what `ensureDaemon` may do only // through the daemon's own answer. Daemons here are loopback stand-ins registered by real children, @@ -107,10 +111,17 @@ test('attach reports a missing daemon without starting one', async (t) => { const error = await failureOf(sendToDaemon(request(stateDir), { daemonLifecycle: 'attach' })); assert.equal(error.details?.reason, 'daemon-not-running'); + assert.match(String(error.details?.hint), /agent-device devices --state-dir /); assert.equal(spawn.mock.calls.length, 0); assert.equal(fs.existsSync(paths.infoPath), false); }); +test('the not-running hint leaves out --state-dir for the default state directory', () => { + const error = daemonNotRunningError(resolveDaemonPaths(undefined).baseDir); + + assert.match(String(error.details?.hint), /\(agent-device devices, for example\)/); +}); + test('attach reports an incompatible daemon with both identities and leaves it running', async (t) => { if (!(await withLoopback(t))) return; const { stateDir, paths } = stateDirAndPaths(); @@ -316,6 +327,31 @@ test.for([ } }); +test('ensureDaemon tells a daemon that is still starting up apart from a busy one', async (t) => { + if (!(await withLoopback(t))) return; + const { stateDir, paths } = stateDirAndPaths(); + const stale = await startScriptedDaemon( + busy({ startingUp: true, inFlightRequests: 0, sessions: 0, repairSessions: 0, leases: 0 }), + ); + const { child } = await registerDaemon(paths, { + httpPort: stale.port, + version: '0.0.0-stale', + capabilities: ['quiesce'], + }); + + try { + const error = await failureOf(ensureDaemon({ stateDir, daemonTransport: 'http' })); + + assert.equal(error.details?.reason, 'daemon-busy'); + assert.equal(error.details?.startingUp, true); + assert.match(String(error.details?.hint), /still starting up\. Retry in a moment/); + assert.equal(isProcessAlive(child.pid), true); + assert.equal(spawn.mock.calls.length, 0); + } finally { + await closeLoopbackServer(stale.server); + } +}); + test('ensureDaemon takes a refusal from its typed reason, not from its wording', async (t) => { if (!(await withLoopback(t))) return; const { stateDir, paths } = stateDirAndPaths(); diff --git a/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts b/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts index ff415b9838..d2eebf6867 100644 --- a/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts +++ b/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts @@ -106,7 +106,7 @@ export async function startScriptedSocketDaemon( export const okResult = (data: Record) => ({ ok: true, data }); export const draining = okResult({ outcome: 'draining' }); -export const busy = (census: Record) => ({ +export const busy = (census: Record) => ({ ok: false, error: { code: 'COMMAND_FAILED', diff --git a/src/daemon-client/daemon-client-ensure.ts b/src/daemon-client/daemon-client-ensure.ts index 3016b1b9ec..80fa42746a 100644 --- a/src/daemon-client/daemon-client-ensure.ts +++ b/src/daemon-client/daemon-client-ensure.ts @@ -127,10 +127,11 @@ async function replaceIncompatible( } const outcome = await quiesceOrRetireUnresponsive(info, settings); if (outcome === 'gone') return { kind: 'judge-again' }; - if (outcome === 'draining') await awaitDrainedDaemon(info, startTime, settings); + const drained = + outcome === 'draining' ? await awaitDrainedDaemon(info, startTime, settings) : outcome; // A daemon force-retired for answering nothing is reported like an unreachable one: it was never // shown to be idle, so "incompatible" would overstate what happened to it. - const reason = outcome === 'retired' ? 'missing' : 'incompatible'; + const reason = drained === 'retired' ? 'missing' : 'incompatible'; return await start(settings, onStart, reason, daemonIdentityOf(info)); } @@ -149,7 +150,7 @@ async function awaitDrainedDaemon( info: DaemonInfo, startTime: string, settings: DaemonClientSettings, -): Promise { +): Promise<'left' | 'retired'> { const { exited } = await waitForDaemonExit( { pid: info.pid, startTime }, { timeoutMs: DRAIN_DEADLINE_MS, pollMs: DRAIN_POLL_MS }, @@ -160,11 +161,11 @@ async function awaitDrainedDaemon( paths: settings.paths, observed: { pid: info.pid, startTime }, }); - return; + return 'left'; } if (await answersLivenessProbe(info)) throw await daemonDrainTimeoutError(info, settings); const retirement = await stopDaemonForTakeover(info, settings.paths, 'force'); - if (retirement.status !== 'retained') return; + if (retirement.status !== 'retained') return 'retired'; // The identity check could not prove the live process is the daemon it observed (an embedded host // runs no daemon entry; a start time or command line may also be unreadable), so nothing was // signaled. diff --git a/src/daemon-client/daemon-client-ensuring.ts b/src/daemon-client/daemon-client-ensuring.ts index 70553e5c6a..24eded8866 100644 --- a/src/daemon-client/daemon-client-ensuring.ts +++ b/src/daemon-client/daemon-client-ensuring.ts @@ -1,7 +1,7 @@ -import { AppError } from '@agent-device/kernel/errors'; import type { DaemonIdentity, EnsureDaemonOptions } from '@agent-device/contracts/client'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import type { sendToDaemon } from './daemon-client.ts'; +import { isDaemonNotRunningError } from './daemon-lifecycle-errors.ts'; type DaemonTransport = typeof sendToDaemon; @@ -66,8 +66,7 @@ export function withEnsuredDaemon( } catch (error) { // The daemon this invocation ensured is gone (idle reap, a crash, another client's stop). // An attach that finds none sent nothing, so ensuring again and sending once is safe. - if (!(error instanceof AppError) || error.details?.reason !== 'daemon-not-running') - throw error; + if (!isDaemonNotRunningError(error)) throw error; // Only the promise this attach relied on is dropped: a concurrent request may already have // replaced it with a fresh one. if (ensured.get(stateDir) === relied) ensured.delete(stateDir); diff --git a/src/daemon-client/daemon-lifecycle-errors.ts b/src/daemon-client/daemon-lifecycle-errors.ts index 434551a99c..e6e32c1e39 100644 --- a/src/daemon-client/daemon-lifecycle-errors.ts +++ b/src/daemon-client/daemon-lifecycle-errors.ts @@ -1,6 +1,7 @@ import type { DaemonIdentity } from '@agent-device/contracts/client'; import { DAEMON_QUIESCE_REASONS, + resolveDaemonPaths, type DaemonQuiesceCensus, } from '@agent-device/daemon-contracts/daemon-resolution'; import { AppError } from '@agent-device/kernel/errors'; @@ -57,14 +58,22 @@ export function daemonContendedError(): AppError { } export function daemonNotRunningError(stateDir: string, registered?: DaemonInfo): AppError { + const stateDirFlag = + stateDir === resolveDaemonPaths(undefined).baseDir + ? '' + : ` --state-dir ${shellQuoteIfNeeded(stateDir)}`; return new AppError('COMMAND_FAILED', 'No daemon is running for this state directory.', { reason: DAEMON_LIFECYCLE_REASONS.notRunning, stateDir, ...(registered ? { daemon: daemonIdentityOf(registered) } : {}), - hint: `Start one with a local agent-device command that uses the daemon (devices, for example), or with ensureDaemon({ stateDir: ${JSON.stringify(stateDir)} }) from the SDK, before attaching.${registered ? ` The registered daemon (pid ${registered.pid}) does not answer; ${daemonStopCommand(stateDir)} clears it.` : ''}`, + hint: `Start one with a local agent-device command that uses the daemon (agent-device devices${stateDirFlag}, for example), or with ensureDaemon({ stateDir: ${JSON.stringify(stateDir)} }) from the SDK, before attaching.${registered ? ` The registered daemon (pid ${registered.pid}) does not answer; ${daemonStopCommand(stateDir)} clears it.` : ''}`, }); } +export function isDaemonNotRunningError(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === DAEMON_LIFECYCLE_REASONS.notRunning; +} + export async function daemonIncompatibleError(params: { info: DaemonInfo; mismatch: DaemonTakeoverCause | 'newer' | 'no-quiesce'; From 5017aa2fd9b0d517225304870a5314df0a6b382f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 13:45:44 +0200 Subject: [PATCH 3/5] docs(daemon-client): describe the force-retired drain in onStart and assert the stop command in the startup hint test Co-Authored-By: Claude Sonnet 5.5 --- packages/contracts/src/client-connection.ts | 3 ++- .../__tests__/daemon-client-explicit-lifecycle.test.ts | 1 + website/docs/docs/client-api.md | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/contracts/src/client-connection.ts b/packages/contracts/src/client-connection.ts index bcb2239c03..4d8599dcc8 100644 --- a/packages/contracts/src/client-connection.ts +++ b/packages/contracts/src/client-connection.ts @@ -52,7 +52,8 @@ export type EnsureDaemonOptions = { * usable daemon was registered; `incompatible` means a daemon this client cannot use was * replaced after it agreed to leave. `previous` is the daemon that was replaced, when there was * one: a `missing` start with a `previous` replaced a registered daemon that answered nothing, - * either unreachable or hung and force-retired. + * either unreachable or hung and force-retired, or one that agreed to leave but outlasted its drain + * and was force-retired. */ onStart?: (reason: 'missing' | 'incompatible', previous?: DaemonIdentity) => void; }; diff --git a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts index fc050514bf..f885714e0b 100644 --- a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts @@ -345,6 +345,7 @@ test('ensureDaemon tells a daemon that is still starting up apart from a busy on assert.equal(error.details?.reason, 'daemon-busy'); assert.equal(error.details?.startingUp, true); assert.match(String(error.details?.hint), /still starting up\. Retry in a moment/); + assert.match(String(error.details?.hint), /agent-device daemon stop --state-dir /); assert.equal(isProcessAlive(child.pid), true); assert.equal(spawn.mock.calls.length, 0); } finally { diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index c0712f2c54..903af2505e 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -109,7 +109,7 @@ await ensureDaemon({ const client = createAgentDeviceClient({ stateDir, daemonLifecycle: 'attach' }); ``` -`ensureDaemon()` starts a daemon when none runs. It replaces a daemon that does not match the client only if that daemon holds no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise it fails and leaves the daemon alone. Two cases retire a daemon without that proof, because it cannot be asked: a registered daemon that answers nothing on any transport, and one that times out on the quiesce request and then fails repeated liveness probes. A daemon that accepted the quiesce request is never signaled while it drains: the call waits up to 60 s for it to exit and clear its registration. One still running after that fails repeated liveness probes and is force-retired when its process is provably a daemon; otherwise the call fails with `daemon-drain-timeout`. It returns `{ stateDir, daemon, started }`; `daemon` holds the pid, version and code identity, and no credentials. `onStart` runs when the call started a daemon: `'incompatible'` when it replaced a daemon that agreed to leave, with that daemon as `previous`, and `'missing'` otherwise. A `'missing'` start that also has a `previous` replaced a registered daemon that answered nothing. +`ensureDaemon()` starts a daemon when none runs. It replaces a daemon that does not match the client only if that daemon holds no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise it fails and leaves the daemon alone. Two cases retire a daemon without that proof, because it cannot be asked: a registered daemon that answers nothing on any transport, and one that times out on the quiesce request and then fails repeated liveness probes. A daemon that accepted the quiesce request is never signaled while it drains: the call waits up to 60 s for it to exit and clear its registration. One still running after that fails repeated liveness probes and is force-retired when its process is provably a daemon; otherwise the call fails with `daemon-drain-timeout`. It returns `{ stateDir, daemon, started }`; `daemon` holds the pid, version and code identity, and no credentials. `onStart` runs when the call started a daemon: `'incompatible'` when it replaced a daemon that agreed to leave, with that daemon as `previous`, and `'missing'` otherwise. A `'missing'` start that also has a `previous` replaced a registered daemon that answered nothing, or one that accepted the request to leave and was force-retired after outlasting its drain. These failures carry a typed `error.details.reason`: From 11200c07c33cb697d20d279657f7b921ab848405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 14:14:12 +0200 Subject: [PATCH 4/5] docs(daemon-client): separate daemon-drain-timeout from an unconfirmed force retirement Co-Authored-By: Claude Sonnet 5.5 --- website/docs/docs/client-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 903af2505e..50fbf0e696 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -109,7 +109,7 @@ await ensureDaemon({ const client = createAgentDeviceClient({ stateDir, daemonLifecycle: 'attach' }); ``` -`ensureDaemon()` starts a daemon when none runs. It replaces a daemon that does not match the client only if that daemon holds no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise it fails and leaves the daemon alone. Two cases retire a daemon without that proof, because it cannot be asked: a registered daemon that answers nothing on any transport, and one that times out on the quiesce request and then fails repeated liveness probes. A daemon that accepted the quiesce request is never signaled while it drains: the call waits up to 60 s for it to exit and clear its registration. One still running after that fails repeated liveness probes and is force-retired when its process is provably a daemon; otherwise the call fails with `daemon-drain-timeout`. It returns `{ stateDir, daemon, started }`; `daemon` holds the pid, version and code identity, and no credentials. `onStart` runs when the call started a daemon: `'incompatible'` when it replaced a daemon that agreed to leave, with that daemon as `previous`, and `'missing'` otherwise. A `'missing'` start that also has a `previous` replaced a registered daemon that answered nothing, or one that accepted the request to leave and was force-retired after outlasting its drain. +`ensureDaemon()` starts a daemon when none runs. It replaces a daemon that does not match the client only if that daemon holds no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise it fails and leaves the daemon alone. Two cases retire a daemon without that proof, because it cannot be asked: a registered daemon that answers nothing on any transport, and one that times out on the quiesce request and then fails repeated liveness probes. A daemon that accepted the quiesce request is never signaled while it drains: the call waits up to 60 s for it to exit and clear its registration. One still running after that fails repeated liveness probes and is force-retired when its process is provably a daemon. A daemon that still answers, or whose process cannot be proven to be a daemon, fails the call with `daemon-drain-timeout`; a force retirement that cannot be confirmed fails it with `daemon_retirement_unconfirmed`. It returns `{ stateDir, daemon, started }`; `daemon` holds the pid, version and code identity, and no credentials. `onStart` runs when the call started a daemon: `'incompatible'` when it replaced a daemon that agreed to leave, with that daemon as `previous`, and `'missing'` otherwise. A `'missing'` start that also has a `previous` replaced a registered daemon that answered nothing, or one that accepted the request to leave and was force-retired after outlasting its drain. These failures carry a typed `error.details.reason`: From 0fa23731a76a2503362bdb9a335c863b939a7257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 15:52:57 +0200 Subject: [PATCH 5/5] fix(daemon-client): always name --state-dir in the not-running hint The default-directory comparison ignored AGENT_DEVICE_STATE_DIR, so a caller passing the default directory while the shell exports that variable got a hint that starts a daemon elsewhere. The retirement-unconfirmed error factory moves to daemon-lifecycle-errors.ts. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 +- .../daemon-client-explicit-lifecycle.test.ts | 12 +-------- src/daemon-client/daemon-client-ensure.ts | 2 +- src/daemon-client/daemon-client-lifecycle.ts | 17 ++++-------- src/daemon-client/daemon-lifecycle-errors.ts | 26 ++++++++++++++----- 5 files changed, 27 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e7d8d8b53..c99ca7fd94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- The `daemon-not-running` hint now names `--state-dir ` when the state directory is not the default, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`. +- The `daemon-not-running` hint now always names `--state-dir `, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`. - Fix `ensureDaemon()`, the CLI and MCP killing a daemon they had just asked to leave. After the daemon answers the quiesce request with `draining`, the client now waits up to 60 s for it to exit by itself and clean up its registration, so plugin disposal and lease finalization finish (on Windows a signal ends the process outright, and on POSIX a drain longer than 3 s was killed). A daemon still running at the deadline that fails its liveness probes is force-retired; one that still answers is left running and the call fails with `details.reason: 'daemon-drain-timeout'`, with a hint naming `agent-device daemon stop --state-dir `. - `@agent-device/testmu` is now a version 2 provider plugin. `import testmu from '@agent-device/testmu'` gives a factory, `testmu({ credentials: { username, accessKey } })`, that reads those credentials instead of `LT_USERNAME` and `LT_ACCESS_KEY`; without options it reads the variables as before. The package still declares the version 1 plugin entry, so `agent-device plugins add @agent-device/testmu` keeps working on hosts that predate the version 2 contract. - Provider plugins have a new contract. A package exports a factory at `./agent-device` that returns `{ apiVersion: 2, provider, credentials, connection?, setup(host) }`, typed by `AgentDevicePluginFactory` from `agent-device/plugins`. The store calls the factory with the package's `config.json` options, and a JavaScript host can call it directly. Version 1 plugins (`agentDevicePlugin.apiVersion: 1`) keep loading. diff --git a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts index f885714e0b..30375c6bd5 100644 --- a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts @@ -4,10 +4,7 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { closeLoopbackServer, supportsLoopbackBind } from '@agent-device/test-kit'; import { AppError } from '@agent-device/kernel/errors'; -import { - DAEMON_QUIESCE_COMMAND, - resolveDaemonPaths, -} from '@agent-device/daemon-contracts/daemon-resolution'; +import { DAEMON_QUIESCE_COMMAND } from '@agent-device/daemon-contracts/daemon-resolution'; import { DAEMON_STARTUP_EXIT_CODES } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { isProcessAlive } from '@agent-device/host-kit/process'; import { readVersion } from '@agent-device/host-kit/version'; @@ -42,7 +39,6 @@ import { sleep } from '@agent-device/host-kit/retry'; import * as transport from '../daemon-client-transport.ts'; import { sendToDaemon } from '../daemon-client.ts'; import { ensureDaemon } from '../daemon-client-ensure.ts'; -import { daemonNotRunningError } from '../daemon-lifecycle-errors.ts'; // The explicit lifecycle: what an attach client may never do, and what `ensureDaemon` may do only // through the daemon's own answer. Daemons here are loopback stand-ins registered by real children, @@ -116,12 +112,6 @@ test('attach reports a missing daemon without starting one', async (t) => { assert.equal(fs.existsSync(paths.infoPath), false); }); -test('the not-running hint leaves out --state-dir for the default state directory', () => { - const error = daemonNotRunningError(resolveDaemonPaths(undefined).baseDir); - - assert.match(String(error.details?.hint), /\(agent-device devices, for example\)/); -}); - test('attach reports an incompatible daemon with both identities and leaves it running', async (t) => { if (!(await withLoopback(t))) return; const { stateDir, paths } = stateDirAndPaths(); diff --git a/src/daemon-client/daemon-client-ensure.ts b/src/daemon-client/daemon-client-ensure.ts index 80fa42746a..504b7e2ae6 100644 --- a/src/daemon-client/daemon-client-ensure.ts +++ b/src/daemon-client/daemon-client-ensure.ts @@ -11,7 +11,6 @@ import { assertDaemonPolicyMatches, observeLocalDaemon, resolveLocalDaemonSettings, - daemonRetirementUnconfirmedError, retireDaemonForTakeover, stopDaemonForTakeover, startLocalDaemon, @@ -26,6 +25,7 @@ import { daemonIdentityOf, daemonDrainTimeoutError, daemonIncompatibleError, + daemonRetirementUnconfirmedError, } from './daemon-lifecycle-errors.ts'; // A racing `ensureDaemon` in another process can retire or replace the daemon this call judged, or diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index be2869c996..9ff93a8891 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -1,7 +1,11 @@ import type { DaemonWireRequest } from '@agent-device/contracts/command'; import type { DaemonLifecycle, EnsureDaemonOptions } from '@agent-device/contracts/client'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; -import { daemonContendedError, daemonStopCommand } from './daemon-lifecycle-errors.ts'; +import { + daemonContendedError, + daemonRetirementUnconfirmedError, + daemonStopCommand, +} from './daemon-lifecycle-errors.ts'; import { attachActiveSessionAddressHint } from './daemon-client-address-hints.ts'; import fs from 'node:fs'; import { processLockHoldsDaemonIdentity } from '@agent-device/daemon-contracts/daemon-registration'; @@ -366,17 +370,6 @@ export async function stopDaemonForTakeover( }); } -export function daemonRetirementUnconfirmedError( - retirement: Extract, - paths: DaemonPaths, -): AppError { - return new AppError('COMMAND_FAILED', 'Daemon replacement could not be confirmed.', { - reason: 'daemon_retirement_unconfirmed', - retirement, - hint: retirement.error?.hint ?? resolveDaemonStartupHint(getDaemonMetadataState(paths), paths), - }); -} - export async function retireDaemonForTakeover( existing: DaemonInfo, paths: DaemonPaths, diff --git a/src/daemon-client/daemon-lifecycle-errors.ts b/src/daemon-client/daemon-lifecycle-errors.ts index e6e32c1e39..3f1e2de7dc 100644 --- a/src/daemon-client/daemon-lifecycle-errors.ts +++ b/src/daemon-client/daemon-lifecycle-errors.ts @@ -1,14 +1,19 @@ import type { DaemonIdentity } from '@agent-device/contracts/client'; import { DAEMON_QUIESCE_REASONS, - resolveDaemonPaths, + type DaemonPaths, type DaemonQuiesceCensus, } from '@agent-device/daemon-contracts/daemon-resolution'; +import type { DaemonRetirementResult } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { AppError } from '@agent-device/kernel/errors'; import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell'; import { readVersion } from '@agent-device/host-kit/version'; import type { DaemonClientSettings } from './daemon-client-lifecycle.ts'; -import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { + getDaemonMetadataState, + resolveDaemonStartupHint, + type DaemonInfo, +} from './daemon-client-metadata.ts'; import { resolveLocalDaemonCodeIdentity, type DaemonTakeoverCause } from './daemon-launch-spec.ts'; // The four ways an explicit lifecycle stops short of a usable daemon. Callers branch on @@ -58,15 +63,22 @@ export function daemonContendedError(): AppError { } export function daemonNotRunningError(stateDir: string, registered?: DaemonInfo): AppError { - const stateDirFlag = - stateDir === resolveDaemonPaths(undefined).baseDir - ? '' - : ` --state-dir ${shellQuoteIfNeeded(stateDir)}`; return new AppError('COMMAND_FAILED', 'No daemon is running for this state directory.', { reason: DAEMON_LIFECYCLE_REASONS.notRunning, stateDir, ...(registered ? { daemon: daemonIdentityOf(registered) } : {}), - hint: `Start one with a local agent-device command that uses the daemon (agent-device devices${stateDirFlag}, for example), or with ensureDaemon({ stateDir: ${JSON.stringify(stateDir)} }) from the SDK, before attaching.${registered ? ` The registered daemon (pid ${registered.pid}) does not answer; ${daemonStopCommand(stateDir)} clears it.` : ''}`, + hint: `Start one with a local agent-device command that uses the daemon (agent-device devices --state-dir ${shellQuoteIfNeeded(stateDir)}, for example), or with ensureDaemon({ stateDir: ${JSON.stringify(stateDir)} }) from the SDK, before attaching.${registered ? ` The registered daemon (pid ${registered.pid}) does not answer; ${daemonStopCommand(stateDir)} clears it.` : ''}`, + }); +} + +export function daemonRetirementUnconfirmedError( + retirement: Extract, + paths: DaemonPaths, +): AppError { + return new AppError('COMMAND_FAILED', 'Daemon replacement could not be confirmed.', { + reason: 'daemon_retirement_unconfirmed', + retirement, + hint: retirement.error?.hint ?? resolveDaemonStartupHint(getDaemonMetadataState(paths), paths), }); }