From 7448765668c5f466c4c2f3e67c551b8bd9436a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 20:27:04 +0200 Subject: [PATCH 1/4] fix(daemon): keep stop and retirement within the observed ownership --- .../daemon-contracts/src/daemon-process.ts | 17 +- .../src/daemon-registration-owner.ts | 86 +++++-- .../src/daemon-registration-status.test.ts | 37 +++ .../src/daemon-registration.ts | 54 +++++ .../src/daemon-resolution-quiesce.test.ts | 68 ++++-- .../daemon-contracts/src/daemon-resolution.ts | 111 +++++++-- .../daemon-contracts/src/daemon-stop.test.ts | 59 +++-- packages/daemon-contracts/src/daemon-stop.ts | 119 +++++++-- .../daemon-runtime-metadata-ownership.test.ts | 21 +- .../http/daemon-runtime-quiesce.test.ts | 65 ++++- .../daemon/src/server/http/daemon-runtime.ts | 11 +- .../sessions/server/daemon-idle-reap.test.ts | 39 ++- .../src/sessions/server/daemon-idle-reap.ts | 112 ++++++--- scripts/clean-daemon.ts | 13 +- ...daemon-registration-embedded-owner.test.ts | 227 ++++++++++++++++++ .../daemon-registration-owner.test.ts | 60 ++++- src/cli/commands/__tests__/daemon.test.ts | 147 +++++++++++- src/cli/commands/daemon.ts | 143 +++++------ src/commands/schema/cli-help.ts | 1 + .../daemon-client-embedded-lifecycle.test.ts | 168 +++++++++++++ .../daemon-client-explicit-lifecycle.test.ts | 14 +- .../__tests__/explicit-lifecycle-fixture.ts | 2 + src/daemon-client/daemon-client-ensure.ts | 29 +-- .../daemon-client-lifecycle-request.ts | 114 +++++++++ src/daemon-client/daemon-client-lifecycle.ts | 15 +- src/daemon-client/daemon-client-metadata.ts | 38 ++- src/daemon-client/daemon-client-quiesce.ts | 74 ------ src/daemon-client/daemon-client-startup.ts | 23 +- src/daemon-client/daemon-client-stop.ts | 58 +++++ src/daemon-client/daemon-client-timeout.ts | 4 +- src/daemon-client/daemon-client.ts | 5 +- src/daemon-client/daemon-lifecycle-errors.ts | 61 ++++- test/integration/smoke-daemon-clean.test.ts | 36 ++- website/docs/docs/client-api.md | 2 + website/docs/docs/commands.md | 2 +- 35 files changed, 1656 insertions(+), 379 deletions(-) create mode 100644 src/__tests__/daemon-registration-embedded-owner.test.ts create mode 100644 src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts create mode 100644 src/daemon-client/daemon-client-lifecycle-request.ts delete mode 100644 src/daemon-client/daemon-client-quiesce.ts create mode 100644 src/daemon-client/daemon-client-stop.ts diff --git a/packages/daemon-contracts/src/daemon-process.ts b/packages/daemon-contracts/src/daemon-process.ts index 28869a01b5..edcc2b5be3 100644 --- a/packages/daemon-contracts/src/daemon-process.ts +++ b/packages/daemon-contracts/src/daemon-process.ts @@ -103,7 +103,22 @@ export type DaemonTerminationResult = }> | Readonly<{ status: 'retained'; - reason: 'missing-start-time' | 'identity-unverified' | 'signal-failed' | 'exit-timeout'; + reason: + | 'missing-start-time' + | 'identity-unverified' + /** + * A host owns this runtime and ends it. Callers that judge a registration answer this instead of + * calling {@link stopDaemonProcess} at all (`stopAndRetireDaemon`); this function signals + * whatever verified identity it is handed and never decides ownership itself. + */ + | 'embedded-owner' + /** + * The record states an owner this build cannot read, so nothing proves this pid is a daemon of + * its own, and signalling it is refused. The other half of the same rule as `embedded-owner`. + */ + | 'owner-unproven' + | 'signal-failed' + | 'exit-timeout'; signal?: NodeJS.Signals; }>; diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 2e452699ad..7717d1a5b1 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -29,6 +29,8 @@ import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import { resolveDaemonPaths, type DaemonCapability, + type DaemonOwnership, + type ObservedDaemonIdentity, type DaemonPaths, type DaemonServerMode, } from './daemon-resolution.ts'; @@ -59,6 +61,7 @@ export type DaemonRegistrationFields = Readonly<{ capabilities?: readonly DaemonCapability[]; /** The provider plugin inventory, so a client can read what this daemon registered. */ plugins?: readonly PluginInventoryEntry[]; + ownership?: DaemonOwnership; }>; type DaemonRegistrationRemoval = | Readonly<{ state: 'removed' }> @@ -333,7 +336,13 @@ function requirePrivateReplayState( export type DaemonRetirementInput = Readonly<{ paths: DaemonPaths; - observed: OwnerIdentity | null; + /** + * The lifetime a caller proved the daemon is, with the owner that record stated at the read. A caller + * that launched the process itself states `detached`, which is what it knows; a caller that read + * `daemon.json` passes what that record stated. Signalling on a guessed owner is the one thing this + * module must not do, so no caller may leave this out. + */ + observed: ObservedDaemonIdentity | null; termTimeoutMs?: number; killTimeoutMs?: number; lockTimeoutMs?: number; @@ -363,6 +372,13 @@ export type DaemonRetirementResult = | 'lock-busy' | 'registration-replaced' | 'metadata-unreadable' + /** A host owns the observed runtime; only that host, or its shutdown request, ends it. */ + | 'embedded-owner' + /** + * The record states an owner this build cannot read, so nothing proves the pid it names is a + * daemon of its own, and signalling it is refused. + */ + | 'owner-unproven' | 'retirement-unconfirmed'; error?: NormalizedError; repairCommitFailure?: RepairCommitFailure; @@ -395,14 +411,24 @@ export async function stopAndRetireDaemon( error: normalizeError(error), }; } + // Signalling needs a proven owner, and the observation carries the one its record stated: a record + // naming a host describes a runtime that host ends, and a record stating an owner this build cannot + // read says nothing about what is safe to end. Only a proven detached daemon is signaled. + const owner = observed?.ownership; const retirement = retireObservedDaemon( { ...input, observed }, - (identity) => - stopDaemonProcess(identity, { - mode: input.mode, - termTimeoutMs: input.termTimeoutMs ?? 3_000, - killTimeoutMs: input.killTimeoutMs ?? 1_000, - }), + observed && owner !== 'detached' + ? (identity) => + proveExitWithoutSignaling( + identity, + owner === 'embedded' ? 'embedded-owner' : 'owner-unproven', + ) + : (identity) => + stopDaemonProcess(identity, { + mode: input.mode, + termTimeoutMs: input.termTimeoutMs ?? 3_000, + killTimeoutMs: input.killTimeoutMs ?? 1_000, + }), owned, input.startupJoinTimeoutMs ?? 1_000, ); @@ -415,8 +441,8 @@ export async function stopAndRetireDaemon( function confirmOwnedStartupIdentity( owned: PrivateReplayState, - observed: OwnerIdentity | null, -): OwnerIdentity | null { + observed: ObservedDaemonIdentity | null, +): ObservedDaemonIdentity | null { if (!observed) { if (owned.startups.length === 0) return null; throw unownedStartupError(); @@ -424,7 +450,7 @@ function confirmOwnedStartupIdentity( const startup = owned.startups.find(({ launch }) => launch.pid === observed.pid); if (!startup) throw unownedStartupError(); const { launch } = startup; - const identity = observed.startTime ? observed : launch.readIdentity(); + const identity = observed.startTime ? observed : { ...observed, ...launch.readIdentity() }; if (launch.startTime && launch.startTime === identity.startTime) return identity; throw unownedStartupError(); } @@ -439,13 +465,33 @@ function unownedStartupError(): AppError { export async function recoverAbandonedDaemonRegistration( input: DaemonRetirementInput, ): Promise { - return await retireObservedDaemon(input, async (identity) => { - if (!identity.startTime?.trim()) return { status: 'retained', reason: 'missing-start-time' }; - const confirmed = { pid: identity.pid, startTime: identity.startTime }; - return (await waitForDaemonExit(confirmed, { timeoutMs: 0 })).exited - ? { status: 'exited', identity: confirmed, mode: 'already-exited' } - : { status: 'retained', reason: 'identity-unverified' }; - }); + return await retireObservedDaemon(input, (identity) => proveExitWithoutSignaling(identity)); +} + +/** + * Termination refusals that name an owner's rule rather than a lifetime this caller could not prove: + * both mean "nobody may signal this process", and both travel to the caller at the top level. + */ +function isOwnerRefusal( + reason: Extract['reason'], +): reason is 'embedded-owner' | 'owner-unproven' { + return reason === 'embedded-owner' || reason === 'owner-unproven'; +} + +/** + * Proves `observed` is gone and signals nothing, which is what a lifecycle path may do to a process + * it does not own: an abandoned registration, or one a host owns and ends itself. A lifetime whose + * exit cannot be proved stays exactly where it is. + */ +async function proveExitWithoutSignaling( + observed: OwnerIdentity, + reason: 'identity-unverified' | 'embedded-owner' | 'owner-unproven' = 'identity-unverified', +): Promise { + if (!observed.startTime?.trim()) return { status: 'retained', reason: 'missing-start-time' }; + const identity = { pid: observed.pid, startTime: observed.startTime }; + return (await waitForDaemonExit(identity, { timeoutMs: 0 })).exited + ? { status: 'exited', identity, mode: 'already-exited' } + : { status: 'retained', reason }; } async function retireObservedDaemon( @@ -468,6 +514,12 @@ async function retireObservedDaemon( const joined = await joinOwnedStartups(owned, startupJoinTimeoutMs); if (stopFailure) return { status: 'retained', reason: 'stop-failed', removedInfo: false, error: stopFailure }; + // A terminate step that refused because the record's owner ends this runtime, or because the record + // could not prove one, says so at the top: a caller must be able to tell "nobody may signal this + // process" from "this daemon is hung". + if (termination?.status === 'retained' && isOwnerRefusal(termination.reason)) { + return { status: 'retained', reason: termination.reason, termination, removedInfo: false }; + } if (termination && termination.status !== 'exited') return { status: 'retained', reason: 'exit-unconfirmed', termination, removedInfo: false }; if (!joined) { diff --git a/packages/daemon-contracts/src/daemon-registration-status.test.ts b/packages/daemon-contracts/src/daemon-registration-status.test.ts index 4458549220..684f2da41a 100644 --- a/packages/daemon-contracts/src/daemon-registration-status.test.ts +++ b/packages/daemon-contracts/src/daemon-registration-status.test.ts @@ -66,12 +66,49 @@ test('a live registered daemon reports its identity and the plugin inventory it state: 'running', stateDir, pid: owner.pid, + // A record that predates ownership is a detached daemon, which is what a reader must be able to + // rely on before it decides a stop may signal this pid. + ownership: 'detached', version: '0.21.30', transport: 'http', plugins: [ACTIVE, FAILED, { state: 'unknown' }], }); }); +test('a record published by a host reports the ownership that limits what a stop may do', () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-status-'); + const owner = readCurrentOwnerIdentity(); + publish(stateDir, { + token: 't', + port: 1, + pid: owner.pid, + processStartTime: owner.startTime, + ownership: 'embedded', + }); + + expect(readDaemonStatus(resolveDaemonPaths(stateDir))).toMatchObject({ + state: 'running', + pid: owner.pid, + ownership: 'embedded', + }); +}); + +test('a record that claims an ownership nobody knows reads as an unrecognized one', () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-status-'); + const owner = readCurrentOwnerIdentity(); + publish(stateDir, { + token: 't', + port: 1, + pid: owner.pid, + processStartTime: owner.startTime, + ownership: 'sideways', + }); + + // An ownership value outside the vocabulary says nothing about who owns the process, and `detached` + // is the one reading a caller may act on by signalling it. Status says it cannot recognize the owner. + expect(readDaemonStatus(resolveDaemonPaths(stateDir)).ownership).toBe('unknown'); +}); + test('a record whose process is gone or was replaced is not a running daemon', () => { const stateDir = mkdtempForTestSync('agent-device-daemon-status-'); const owner = readCurrentOwnerIdentity(); diff --git a/packages/daemon-contracts/src/daemon-registration.ts b/packages/daemon-contracts/src/daemon-registration.ts index fb2723e4d0..e1f7ca96f0 100644 --- a/packages/daemon-contracts/src/daemon-registration.ts +++ b/packages/daemon-contracts/src/daemon-registration.ts @@ -8,9 +8,11 @@ import { type OwnerIdentity, } from '@agent-device/host-kit/process'; import { + readDaemonOwnership, readDaemonRecord, readRecordedDaemonTransport, resolveDaemonPaths, + type DaemonOwnership, type DaemonPaths, type DaemonServerMode, } from './daemon-resolution.ts'; @@ -125,6 +127,11 @@ export type DaemonStatus = Readonly<{ pid?: number; version?: string; transport?: DaemonServerMode; + /** + * Who owns the process this record names, and so who may end it. Only a running record can say: a + * stopped one names no live process to own. + */ + ownership?: DaemonOwnership; /** * The provider plugin inventory the running daemon published, as recorded; `readPluginInventory` * validates it where it is shown, so this module stays out of the inventory vocabulary's closure. @@ -171,6 +178,7 @@ export function readDaemonStatus(paths: DaemonPaths): DaemonStatus { state: 'running', stateDir: paths.baseDir, pid: identity.pid, + ownership: readDaemonOwnership(record.ownership), ...(typeof record.version === 'string' ? { version: record.version } : {}), ...(transport ? { transport } : {}), plugins: record.plugins, @@ -181,6 +189,52 @@ function readableStartTime(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value : null; } +/** + * What one read of `daemon.json` proves about the process it names, and the one observation every stop + * route judges. The lifetime and the owner come from the same read on purpose: a stop that routed on one + * record and retired on another could signal a process the record it acted on did not name. + * + * `identity` is null for a readable record that names no process, which leaves nothing to signal. + * `absent` is no record at all, so nothing claims the state directory. `unreadable` is a record that is + * present and cannot be read or parsed: it may describe a runtime that is serving, so a stop refuses it + * rather than reporting that nothing runs. + */ +export type DaemonStopIdentity = Readonly<{ + /** `absent`: no record. `unreadable`: a record whose contents cannot be used, unreadable or malformed. */ + ownership: DaemonOwnership | 'absent' | 'unreadable'; + identity: OwnerIdentity | null; +}>; + +/** + * Reads a state directory's record once for a stop: whether it names a process, and whose process that + * is. A record that states no owner describes the pre-existing detached daemon, which is what every + * record written before the ownership field named. + */ +export function readDaemonStopIdentity(infoPath: string): DaemonStopIdentity { + const read = readDaemonRecord(infoPath); + if (read.status !== 'present') { + return { + ownership: read.status === 'absent' ? 'absent' : 'unreadable', + identity: null, + }; + } + const parsed = parseRegistration(read.record); + const identity = parsed.pid === null ? null : { pid: parsed.pid, startTime: parsed.startTime }; + return { ownership: readDaemonOwnership(read.record.ownership), identity }; +} + +/** + * The lifetime a `--clean` cleanup may walk, from the read that routed the stop. Only a record proving + * the process is a daemon of its own names one, and the cleanup needs a start time to tell that lifetime + * from whatever holds its pid now. + */ +export function readStopCleanupOwner( + record: DaemonStopIdentity, +): { pid: number; processStartTime: string } | null { + if (record.ownership !== 'detached' || !record.identity?.startTime) return null; + return { pid: record.identity.pid, processStartTime: record.identity.startTime }; +} + /** * #2031: positive proof that `owner` is no longer the daemon serving its own * state dir, and so can never be asked to release what it holds — its sessions diff --git a/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts b/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts index 67897417b5..b603a631bb 100644 --- a/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts +++ b/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts @@ -1,6 +1,13 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { readDaemonQuiesceCensus, readDaemonQuiesceFence } from './daemon-resolution.ts'; +import { + buildDaemonLifecycleRequest, + DAEMON_LIFECYCLE_COMMANDS, + readDaemonLifecycleCensus, + readDaemonLifecycleFence, + readDaemonLifecycleMode, + readDaemonOwnership, +} from './daemon-resolution.ts'; test('a census is read only when every count is a non-negative integer', () => { const census = { @@ -10,30 +17,63 @@ test('a census is read only when every count is a non-negative integer', () => { repairSessions: 2, leases: 3, }; - assert.deepEqual(readDaemonQuiesceCensus({ reason: 'daemon-busy', ...census }), census); + assert.deepEqual(readDaemonLifecycleCensus({ reason: 'daemon-busy', ...census }), census); - for (const malformed of [-1, 1.5, '1', Number.NaN, Number.POSITIVE_INFINITY, null]) { - assert.equal(readDaemonQuiesceCensus({ ...census, leases: malformed }), undefined); + // 2 ** 53 is an integer `Number.isInteger` accepts and arithmetic on it gets wrong, so the census + // reader has to be the one that refuses it. + for (const malformed of [-1, 1.5, '1', Number.NaN, Number.POSITIVE_INFINITY, 2 ** 53, null]) { + assert.equal(readDaemonLifecycleCensus({ ...census, leases: malformed }), undefined); } - assert.equal(readDaemonQuiesceCensus({ sessions: 1 }), undefined); + assert.equal(readDaemonLifecycleCensus({ sessions: 1 }), undefined); // Only a daemon that is starting up says so; an older one omits the field. - assert.equal(readDaemonQuiesceCensus({ ...census, startingUp: true })?.startingUp, true); + assert.equal(readDaemonLifecycleCensus({ ...census, startingUp: true })?.startingUp, true); const { startingUp: _omitted, ...withoutStartingUp } = census; - assert.equal(readDaemonQuiesceCensus(withoutStartingUp)?.startingUp, false); - assert.equal(readDaemonQuiesceCensus({ ...census, startingUp: 'yes' }), undefined); - assert.equal(readDaemonQuiesceCensus(undefined), undefined); + assert.equal(readDaemonLifecycleCensus(withoutStartingUp)?.startingUp, false); + assert.equal(readDaemonLifecycleCensus({ ...census, startingUp: 'yes' }), undefined); + assert.equal(readDaemonLifecycleCensus(undefined), undefined); }); test('a fence names an integer pid and a start time or null, nothing else', () => { - assert.deepEqual(readDaemonQuiesceFence({ pid: 7, processStartTime: 'Fri' }), { + assert.deepEqual(readDaemonLifecycleFence({ pid: 7, processStartTime: 'Fri' }), { pid: 7, processStartTime: 'Fri', }); - assert.deepEqual(readDaemonQuiesceFence({ pid: 7, processStartTime: null }), { + assert.deepEqual(readDaemonLifecycleFence({ pid: 7, processStartTime: null }), { pid: 7, processStartTime: null, }); - assert.equal(readDaemonQuiesceFence({ pid: 7 }), undefined); - assert.equal(readDaemonQuiesceFence({ pid: 7.5, processStartTime: null }), undefined); - assert.equal(readDaemonQuiesceFence('7'), undefined); + assert.equal(readDaemonLifecycleFence({ pid: 7 }), undefined); + assert.equal(readDaemonLifecycleFence({ pid: 7.5, processStartTime: null }), undefined); + assert.equal(readDaemonLifecycleFence('7'), undefined); +}); + +test('a quiesce carries the original instance fence', () => { + const fence = { pid: 7, processStartTime: 'Fri Oct 17 09:00:00 2026' }; + for (const mode of ['quiesce'] as const) { + const request = buildDaemonLifecycleRequest({ mode, token: 't0ken', fence, requestId: 'r-1' }); + assert.equal(request.command, DAEMON_LIFECYCLE_COMMANDS[mode]); + assert.deepEqual(request.input, fence); + assert.equal(request.meta?.requestId, 'r-1'); + assert.equal(readDaemonLifecycleMode(request.command), mode); + } + // A request that is not one of the two is dispatched like any other command, including a name the + // lookup would answer for through the object prototype. + assert.equal(readDaemonLifecycleMode('open'), undefined); + assert.equal(readDaemonLifecycleMode('toString'), undefined); + assert.equal(readDaemonLifecycleMode('__proto__'), undefined); +}); + +test('ownership names who may end the process, and an older record is detached', () => { + assert.equal(readDaemonOwnership('embedded'), 'embedded'); + assert.equal(readDaemonOwnership('detached'), 'detached'); + // An absent field is every record written before this field existed, and those are daemons that are + // their own process. + for (const absent of [undefined]) { + assert.equal(readDaemonOwnership(absent), 'detached'); + } + // Anything else is its own state: a caller that means to signal a pid reads `unknown` as a thing it + // must not do, not as permission to treat the record like a detached daemon's. + for (const unrecognized of [null, 'owned', 7, '', 'EMBEDDED', 2 ** 53]) { + assert.equal(readDaemonOwnership(unrecognized), 'unknown', String(unrecognized)); + } }); diff --git a/packages/daemon-contracts/src/daemon-resolution.ts b/packages/daemon-contracts/src/daemon-resolution.ts index 1d95738b18..d4c0ddd818 100644 --- a/packages/daemon-contracts/src/daemon-resolution.ts +++ b/packages/daemon-contracts/src/daemon-resolution.ts @@ -7,6 +7,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { type EnvMap } from '@agent-device/kernel/source-value'; import type { DaemonWireRequest } from '@agent-device/contracts/command'; +import type { OwnerIdentity } from '@agent-device/host-kit/process'; import type { DaemonServerMode, DaemonTransportPreference } from '@agent-device/kernel/contracts'; export type { DaemonServerMode, DaemonTransportPreference }; @@ -146,35 +147,108 @@ export function resolveDaemonTransportPreference( return 'auto'; } -// The quiesce request is how a client asks an idle daemon to leave so a compatible one can take -// its state directory. The client and the daemon share this vocabulary, so neither owns it. - /** What a daemon can do beyond the base RPC contract, published in its `daemon.json`. */ export const DAEMON_CAPABILITIES = ['quiesce'] as const; export type DaemonCapability = (typeof DAEMON_CAPABILITIES)[number]; -export const DAEMON_QUIESCE_COMMAND = 'daemon_quiesce'; +/** + * Who owns the process a registration names, and so who may end it. + * + * - `detached`: the daemon is its own process, which a stop may signal. This is what a record that + * predates the field names. + * - `embedded`: the host that started the runtime owns the process. Nothing in the central + * lifecycle signals that PID, not even hung recovery or a forced stop: the host closes the + * runtime through its own close, and an external stop refuses it. + * - `unknown`: the record states an owner this build does not recognize, so no process may be + * signaled on its strength. Signalling on a guess could end a host process, and a value this + * build cannot read is exactly the case where the safe answer is to do nothing. + */ +const DAEMON_OWNERSHIPS = ['detached', 'embedded', 'unknown'] as const; +export type DaemonOwnership = (typeof DAEMON_OWNERSHIPS)[number]; + +/** + * The ownership a process can state about *itself*: `unknown` is what a reader answers for a record it + * cannot place, never what a runtime knows about its own process. So a runtime starts as detached or + * embedded, and the refusal for a record this build cannot read belongs to the code that reads the + * record — see {@link readDaemonOwnership}. + */ +export type DaemonRuntimeOwnership = Exclude; + +/** + * The lifetime a lifecycle path has proved the daemon on a state directory is, together with the owner + * its record stated at that read. Ownership travels inside the observation because a pid only says what + * is safe to end once its owner is known, and two reads of `daemon.json` can name different pairs. + */ +export type ObservedDaemonIdentity = OwnerIdentity & { + ownership: DaemonOwnership; +}; + +/** + * The ownership a `daemon.json` field records. An absent field is the pre-existing detached daemon — + * every record written before #3757 states no owner and describes a daemon that is its own process. + * A field that states something else is `unknown`, and is never folded into `detached`: the two read + * the same way to a caller that is about to signal a pid, and only one of them is safe to act on. + */ +export function readDaemonOwnership(value: unknown): DaemonOwnership { + return DAEMON_OWNERSHIPS.includes(value as DaemonOwnership) + ? (value as DaemonOwnership) + : value === undefined + ? 'detached' + : 'unknown'; +} + +/** The command carrying the fenced lifecycle request. */ +export const DAEMON_LIFECYCLE_COMMANDS = { + /** Stop if idle: refuses while the daemon holds a request, a session, or a lease. */ + quiesce: 'daemon_quiesce', +} as const; +export type DaemonLifecycleMode = keyof typeof DAEMON_LIFECYCLE_COMMANDS; + +/** + * The lifecycle mode `command` names, or `undefined` for every request that is not one. Compared + * explicitly, because a lookup in an object would answer for a command like `toString` too. + */ +export function readDaemonLifecycleMode(command: string): DaemonLifecycleMode | undefined { + switch (command) { + case DAEMON_LIFECYCLE_COMMANDS.quiesce: + return 'quiesce'; + default: + return undefined; + } +} /** The reason a daemon gives every request it refuses once it has begun to leave. */ export const DAEMON_SHUTTING_DOWN_REASON = 'daemon_shutting_down'; -export const DAEMON_QUIESCE_REASONS = { +/** Why a daemon refuses a lifecycle request: one entry per rule the admission seam enforces. */ +export const DAEMON_LIFECYCLE_REASONS = { + /** A quiesce found the daemon holding work, so nothing was accepted. */ busy: 'daemon-busy', + /** The request names a different daemon instance than the one answering it. */ instanceMismatch: 'daemon-instance-mismatch', + /** A quiesce was refused because a host owns this runtime and ends it through its own close. */ + ownerEmbedded: 'daemon-owner-embedded', } as const; -/** The daemon instance a quiesce request is addressed to; any other instance refuses it. */ -export type DaemonQuiesceFence = Readonly<{ +/** + * The refusal for a `daemon.json` whose owner field this build cannot place. No daemon answers it: a + * runtime always knows whether it owns itself, so `unknown` is a verdict about a file, and only a reader + * can reach it. + */ +export const DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON = 'daemon-owner-unrecognized'; + +/** The daemon instance a lifecycle request is addressed to; any other instance refuses it. */ +export type DaemonLifecycleFence = Readonly<{ pid: number; /** Null when the daemon could not prove its start time, as its `daemon.json` records. */ processStartTime: string | null; }>; /** - * What a daemon holds. It leaves a quiesce request only when all four counts are zero and it is - * not still starting up, because before that it has no drain path to run. + * What a daemon holds, and the census a busy refusal carries so a caller knows what to wait for. + * A quiesce leaves only when all four counts are zero and it is not still starting up. */ -export type DaemonQuiesceCensus = Readonly<{ +export type DaemonLifecycleCensus = Readonly<{ /** True from process start until the daemon can drain. */ startingUp: boolean; inFlightRequests: number; @@ -183,23 +257,28 @@ export type DaemonQuiesceCensus = Readonly<{ leases: number; }>; -export function buildDaemonQuiesceRequest(params: { +/** + * The request that asks the fenced instance to leave. Both modes carry the fence in `input`, so a + * daemon refuses a request addressed to a predecessor that reused its pid. + */ +export function buildDaemonLifecycleRequest(params: { + mode: DaemonLifecycleMode; token: string; - fence: DaemonQuiesceFence; + fence: DaemonLifecycleFence; requestId: string; }): DaemonWireRequest { - const { token, fence, requestId } = params; + const { mode, token, fence, requestId } = params; return { token, session: 'default', - command: DAEMON_QUIESCE_COMMAND, + command: DAEMON_LIFECYCLE_COMMANDS[mode], positionals: [], input: { pid: fence.pid, processStartTime: fence.processStartTime }, meta: { requestId }, }; } -export function readDaemonQuiesceFence(input: unknown): DaemonQuiesceFence | undefined { +export function readDaemonLifecycleFence(input: unknown): DaemonLifecycleFence | undefined { if (typeof input !== 'object' || input === null) return undefined; const { pid, processStartTime } = input as Record; if (!Number.isInteger(pid)) return undefined; @@ -212,7 +291,7 @@ export function readDaemonCapabilities(value: unknown): readonly DaemonCapabilit return DAEMON_CAPABILITIES.filter((capability) => value.includes(capability)); } -export function readDaemonQuiesceCensus(details: unknown): DaemonQuiesceCensus | undefined { +export function readDaemonLifecycleCensus(details: unknown): DaemonLifecycleCensus | undefined { if (typeof details !== 'object' || details === null) return undefined; const { startingUp, inFlightRequests, sessions, repairSessions, leases } = details as Record< string, diff --git a/packages/daemon-contracts/src/daemon-stop.test.ts b/packages/daemon-contracts/src/daemon-stop.test.ts index e5003cb030..64d47e4dc4 100644 --- a/packages/daemon-contracts/src/daemon-stop.test.ts +++ b/packages/daemon-contracts/src/daemon-stop.test.ts @@ -7,7 +7,7 @@ vi.mock('./daemon-registration-owner.ts', () => ({ stopAndRetireDaemon: mocks.stopAndRetireDaemon, })); -import { resolveDaemonPaths } from './daemon-resolution.ts'; +import { resolveDaemonPaths, type ObservedDaemonIdentity } from './daemon-resolution.ts'; import { stopDaemon } from './daemon-stop.ts'; afterEach(() => { @@ -18,35 +18,45 @@ function createDaemonPaths(): ReturnType { const stateDir = mkdtempForTestSync('agent-device-daemon-stop-'); const paths = resolveDaemonPaths(stateDir); fs.mkdirSync(paths.baseDir, { recursive: true }); + // The record's contents are the caller's business: a stop hands over the lifetime and owner its own + // read proved, so this fixture only has to put a record on disk for a retirement to act on. fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123, processStartTime: 'start-time' })); return paths; } +/** The lifetime the record above names, which is what a caller hands to the retirement it routes. */ +const observed: ObservedDaemonIdentity = { + pid: 123, + startTime: 'start-time', + ownership: 'detached', +}; + function removeDaemonPaths(paths: ReturnType): void { fs.rmSync(paths.baseDir, { recursive: true, force: true }); } -test('reports not-running when daemon metadata is absent', async () => { +test('a caller that read no lifetime reports not-running without retiring', async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-daemon-stop-')); try { - const result = await stopDaemon({ paths }); + const result = await stopDaemon({ paths, observed: null }); expect(result).toMatchObject({ stopped: false, mode: 'not-running' }); } finally { removeDaemonPaths(paths); } + expect(mocks.stopAndRetireDaemon).not.toHaveBeenCalled(); }); test('retained identity verification is reported as failure without known cleanup', async () => { const paths = createDaemonPaths(); mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit('identity-unverified')); - await expect(stopDaemon({ paths })).rejects.toMatchObject({ + await expect(stopDaemon({ paths, observed })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_exit_unconfirmed', terminationReason: 'identity-unverified' }, }); expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith({ paths, - observed: { pid: 123, startTime: 'start-time' }, + observed, mode: 'graceful', termTimeoutMs: 10_000, killTimeoutMs: 2_000, @@ -55,19 +65,23 @@ test('retained identity verification is reported as failure without known cleanu test('missing start-time identity is passed to the owning termination operation', async () => { const paths = createDaemonPaths(); - fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123, processStartTime: ' ' })); mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit('missing-start-time')); - await expect(stopDaemon({ paths })).rejects.toMatchObject({ + await expect( + stopDaemon({ paths, observed: { pid: 123, startTime: null, ownership: 'detached' } }), + ).rejects.toMatchObject({ details: { terminationReason: 'missing-start-time' }, }); expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith( - expect.objectContaining({ paths, observed: { pid: 123, startTime: null } }), + expect.objectContaining({ + paths, + observed: { pid: 123, startTime: null, ownership: 'detached' }, + }), ); }); test('a previously exited verified lifetime is reported as not-running', async () => { mocks.stopAndRetireDaemon.mockResolvedValue(retired('already-exited')); - expect(await stopDaemon({ paths: createDaemonPaths() })).toMatchObject({ + expect(await stopDaemon({ paths: createDaemonPaths(), observed })).toMatchObject({ stopped: false, mode: 'not-running', }); @@ -75,14 +89,15 @@ test('a previously exited verified lifetime is reported as not-running', async ( test('an already released pid without start time remains not-running without cleanup proof', async () => { const paths = createDaemonPaths(); - fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123 })); mocks.stopAndRetireDaemon.mockResolvedValue({ status: 'retained', reason: 'exit-unconfirmed', removedInfo: false, termination: { status: 'not-running' }, }); - expect(await stopDaemon({ paths })).toMatchObject({ + expect( + await stopDaemon({ paths, observed: { pid: 123, startTime: null, ownership: 'detached' } }), + ).toMatchObject({ stopped: false, mode: 'not-running', }); @@ -91,15 +106,17 @@ test('an already released pid without start time remains not-running without cle test('confirmed TERM exit preserves graceful report behavior and configured budgets', async () => { const paths = createDaemonPaths(); mocks.stopAndRetireDaemon.mockResolvedValue(retired('graceful')); - expect(await stopDaemon({ paths, graceTimeoutMs: 11, killTimeoutMs: 7 })).toMatchObject({ - stopped: true, - mode: 'graceful', - cleanupConfidence: 'known', - providerReleases: { pending: [] }, - }); + expect(await stopDaemon({ paths, observed, graceTimeoutMs: 11, killTimeoutMs: 7 })).toMatchObject( + { + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + providerReleases: { pending: [] }, + }, + ); expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith({ paths, - observed: { pid: 123, startTime: 'start-time' }, + observed, mode: 'graceful', termTimeoutMs: 11, killTimeoutMs: 7, @@ -108,7 +125,7 @@ test('confirmed TERM exit preserves graceful report behavior and configured budg test('confirmed KILL exit preserves unknown provider cleanup', async () => { mocks.stopAndRetireDaemon.mockResolvedValue(retired('forced')); - expect(await stopDaemon({ paths: createDaemonPaths() })).toMatchObject({ + expect(await stopDaemon({ paths: createDaemonPaths(), observed })).toMatchObject({ stopped: true, mode: 'forced', cleanupConfidence: 'unknown', @@ -121,7 +138,7 @@ test.each(['signal-failed', 'exit-timeout'])( '%s cannot become a successful stop', async (reason) => { mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit(reason)); - await expect(stopDaemon({ paths: createDaemonPaths() })).rejects.toMatchObject({ + await expect(stopDaemon({ paths: createDaemonPaths(), observed })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_exit_unconfirmed', terminationReason: reason }, }); @@ -162,7 +179,7 @@ test.each(['registration-replaced', 'retirement-unconfirmed'])( logPath: '/retained/daemon.log', }, }); - await expect(stopDaemon({ paths })).rejects.toMatchObject({ + await expect(stopDaemon({ paths, observed })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_retirement_unconfirmed', diff --git a/packages/daemon-contracts/src/daemon-stop.ts b/packages/daemon-contracts/src/daemon-stop.ts index f4a6d67639..4302065416 100644 --- a/packages/daemon-contracts/src/daemon-stop.ts +++ b/packages/daemon-contracts/src/daemon-stop.ts @@ -1,10 +1,13 @@ import { AppError } from '@agent-device/kernel/errors'; import type { DaemonRetirementResult } from './daemon-registration-owner.ts'; -import type { OwnerIdentity } from '@agent-device/host-kit/process'; -import type { DaemonPaths } from './daemon-resolution.ts'; -import { readRegisteredDaemonIdentity } from './daemon-registration.ts'; -import type { DeviceClaimRecord, ProviderReleaseRecord } from './daemon-shutdown-report.ts'; +import type { DaemonPaths, ObservedDaemonIdentity } from './daemon-resolution.ts'; +import type { OwnerIdentity } from '@agent-device/host-kit/process'; +import type { + DaemonShutdownReport, + DeviceClaimRecord, + ProviderReleaseRecord, +} from './daemon-shutdown-report.ts'; const DAEMON_STOP_GRACE_TIMEOUT_MS = 10_000; const DAEMON_STOP_KILL_TIMEOUT_MS = 2_000; @@ -36,25 +39,33 @@ export type DaemonStopResult = { warnings: string[]; }; +/** + * Retires the daemon whose lifetime the caller read, and shapes that retirement as a stop result. + * + * The caller hands over the lifetime and the owner its own read proved, so routing and retirement judge + * one record. Only a detached daemon reaches here: a host owns an embedded runtime and ends it with its + * own close, and a record this build cannot place proves nothing about what is safe to signal. + */ export async function stopDaemon(params: { paths: DaemonPaths; + observed: ObservedDaemonIdentity | null; graceTimeoutMs?: number; killTimeoutMs?: number; }): Promise { - const info = readRegisteredDaemonIdentity(params.paths.infoPath); - if (!info) return notRunningResult(); + const observed = params.observed; + if (!observed) return daemonNotRunningStopResult(); const { stopAndRetireDaemon } = await import('./daemon-registration-owner.ts'); const retirement = await stopAndRetireDaemon({ paths: params.paths, - observed: info, + observed, mode: 'graceful', termTimeoutMs: params.graceTimeoutMs ?? DAEMON_STOP_GRACE_TIMEOUT_MS, killTimeoutMs: params.killTimeoutMs ?? DAEMON_STOP_KILL_TIMEOUT_MS, }); if (retirement.status === 'retained' && retirement.termination?.status === 'not-running') - return notRunningResult(); - if (retirement.status !== 'retired') throw daemonRetirementError(info, retirement); - if (retirement.termination.mode === 'already-exited') return notRunningResult(); + return daemonNotRunningStopResult(); + if (retirement.status !== 'retired') throw daemonRetirementError(observed, retirement); + if (retirement.termination.mode === 'already-exited') return daemonNotRunningStopResult(); if (retirement.termination.mode === 'graceful') { return { stopped: true, @@ -105,15 +116,7 @@ function daemonRetirementError( }); } -export function readDaemonStopIdentity( - infoPath: string, -): { pid: number; processStartTime: string } | null { - const info = readRegisteredDaemonIdentity(infoPath); - if (!info?.startTime) return null; - return { pid: info.pid, processStartTime: info.startTime }; -} - -function notRunningResult(): DaemonStopResult { +function daemonNotRunningStopResult(): DaemonStopResult { return { stopped: false, mode: 'not-running', @@ -126,3 +129,81 @@ function notRunningResult(): DaemonStopResult { warnings: [], }; } + +export function mergeDaemonShutdownReport( + stopped: DaemonStopResult, + report: DaemonShutdownReport | null, +): DaemonStopResult { + if (stopped.mode !== 'graceful' || report) { + return report + ? { + ...stopped, + providerReleases: { status: 'completed', ...report.providerReleases }, + claimsReleased: report.claims.released, + claimsOrphaned: report.claims.orphaned, + claimsSuperseded: report.claims.superseded, + claimsUnattributable: report.claims.unattributable, + warnings: [ + ...stopped.warnings, + ...supersededClaimWarnings(report.claims.superseded), + ...orphanedClaimWarnings(report.claims.orphaned), + ...unattributableClaimWarnings(report.claims.unattributable), + ], + } + : stopped; + } + return { + ...stopped, + cleanupConfidence: 'unknown', + providerReleases: { status: 'unknown', released: [], pending: null }, + warnings: [ + ...stopped.warnings, + 'The graceful shutdown report is unavailable, so provider cleanup state is unknown. Provider allocations may remain active.', + ], + }; +} + +/** A superseded claim is not a failure to report as one, but the operator's + * device is now owned elsewhere, so it must not pass silently. */ +function supersededClaimWarnings(superseded: DaemonStopResult['claimsSuperseded']): string[] { + if (superseded.length === 0) return []; + const devices = superseded.map((claim) => claim.deviceId).join(', '); + return [ + `Another owner had already claimed ${devices} before this daemon released it, so those devices are now owned elsewhere.`, + ]; +} + +/** An orphaned claim keeps holding its device after the daemon is gone, and + * only `device release --stale` or the next open settles it — say so instead + * of leaving the block discoverable through --json alone. + * + * A record whose owner could not be attributed is NOT reported here. Those are what + * `--stale` hides and refuses, so this sentence would send the operator to a route that cannot + * succeed; {@link unattributableClaimWarnings} names the view that does show them. */ +function orphanedClaimWarnings(orphaned: DaemonStopResult['claimsOrphaned']): string[] { + if (orphaned.length === 0) return []; + const devices = orphaned.map((claim) => claim.deviceId).join(', '); + return [ + `Ownership of ${devices} was not released cleanly; the claim now blocks other owners until it is settled. Inspect with: agent-device device status --stale, then release with: agent-device device release --stale.`, + ]; +} + +/** + * What the three provable buckets each get an exact remedy for, this one does not: settling a claim + * always rests on a proof about its owner, and this record names none. So the warning states only + * what is known, points at the view that lists these records — the default one, because `--stale` + * filters records without a decodable owner out — and does not promise a release that would clear + * them. `device release --stale` refuses every record that names no owner, and the next `open` refuses + * to overwrite one. + */ +function unattributableClaimWarnings( + unattributable: DaemonStopResult['claimsUnattributable'], +): string[] { + if (unattributable.length === 0) return []; + const devices = unattributable.map((claim) => claim.deviceId).join(', '); + return [ + `A claim for ${devices} remains whose owner could not be read, so this daemon cannot say whether those devices are free. ` + + 'No device release route settles it: --stale proves staleness from the recorded owner and refuses records that name none, ' + + 'and open refuses to overwrite them. Inspect with: agent-device device status.', + ]; +} diff --git a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts index f77c0fd0e0..084dd43c04 100644 --- a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts @@ -254,16 +254,23 @@ test('both exits tear the watch down before they touch daemon.json', () => { // A watch still armed when the removal runs can report a loss for a record its own process just // deleted. The startup-failure half of this is unreachable from a test — everything after // publication needs a real toolchain — so the invariant is read off the source, the same way the - // arming order above is. Startup rollback runs its steps in reverse, so the registration that - // stops the watch comes after the one that finishes the registration; the close path stops first. + // arming order above is. const source = fs.readFileSync(new URL('./daemon-runtime.ts', import.meta.url), 'utf8'); const stops = [...source.matchAll(/stopMetadataLossWatch\(\);/g)].map((m) => m.index); - const finishes = [...source.matchAll(/await finishDaemonRegistration\(/g)].map((m) => m.index); + // Startup rollback runs its steps in reverse, so the step that stops the watch must be REGISTERED + // after the one that removes the record for it to run first. + const rollbackRemoval = source.indexOf('await finishDaemonRegistration({'); + // The close path: the watch is down before the release, which is the only step that removes the + // record on that exit, can run. + const closeStart = source.indexOf('const closeSteps = async'); + const closeStop = source.indexOf('stopMetadataLossWatch();', closeStart); + const closeRelease = source.indexOf('finishDaemonRegistration({', closeStop); + expect(stops).toHaveLength(2); - expect(finishes).toHaveLength(2); - expect(stops[0]).toBeGreaterThan(finishes[0]!); - expect(stops[1]).toBeGreaterThan(stops[0]!); - expect(finishes[1]).toBeGreaterThan(stops[1]!); + expect(rollbackRemoval).toBeGreaterThan(0); + expect(stops[0]).toBeGreaterThan(rollbackRemoval); + expect(closeStop).toBeGreaterThan(0); + expect(closeRelease).toBeGreaterThan(closeStop); }); test('a plugin whose cleanup failed at shutdown is recorded in daemon.log', async () => { diff --git a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts b/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts index 110b06d9c5..c4f289ec7a 100644 --- a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts @@ -3,7 +3,7 @@ import net from 'node:net'; import { afterEach, expect, test, vi } from 'vitest'; import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; import { - buildDaemonQuiesceRequest, + buildDaemonLifecycleRequest, resolveDaemonPaths, } from '@agent-device/daemon-contracts/daemon-resolution'; import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; @@ -96,17 +96,24 @@ type Booted = { sessionList(): Promise; }; -async function bootDaemon(options: { serverMode?: 'socket' | 'dual' } = {}): Promise { +async function bootDaemon( + options: { + serverMode?: 'socket' | 'dual'; + ownership?: 'embedded' | 'detached'; + idleTimeoutMs?: number; + } = {}, +): Promise { const stateDir = mkdtempForTestSync('agent-device-daemon-quiesce-'); const paths = resolveDaemonPaths(stateDir); const runtime = await startDaemonRuntime({ + ownership: options.ownership, composeDeps: daemonRuntimeDepsFixture(), host: createTestHostEnvironment({ env: { ...process.env, AGENT_DEVICE_STATE_DIR: stateDir, AGENT_DEVICE_DAEMON_SERVER_MODE: options.serverMode ?? 'socket', - AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: String(options.idleTimeoutMs ?? 0), }, }), stderr: { write: () => {} }, @@ -126,7 +133,8 @@ async function bootDaemon(options: { serverMode?: 'socket' | 'dual' } = {}): Pro exit, quiesce: (overrides = {}) => send( - buildDaemonQuiesceRequest({ + buildDaemonLifecycleRequest({ + mode: 'quiesce', token: overrides.token ?? runtime.token, fence: { pid: overrides.pid ?? info.pid, @@ -289,7 +297,8 @@ async function quiesceOverHttp( jsonrpc: '2.0', id: 'http-quiesce', method: 'agent_device.command', - params: buildDaemonQuiesceRequest({ + params: buildDaemonLifecycleRequest({ + mode: 'quiesce', token, fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, requestId: 'http-quiesce', @@ -350,7 +359,8 @@ test('a quiesce asked while the daemon is still starting up is refused as busy, processStartTime?: string; }; - const quiesceRequest = buildDaemonQuiesceRequest({ + const quiesceRequest = buildDaemonLifecycleRequest({ + mode: 'quiesce', token: info.token, fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, requestId: 'startup-quiesce', @@ -376,3 +386,46 @@ test('a quiesce asked while the daemon is still starting up is refused as busy, await starting.then((runtime) => runtime?.close()); } }); + +test('an embedded runtime outlives the idle window a detached daemon reaps itself on', async () => { + // Booted first on purpose: both get the same window, so the embedded one's window ends no later than + // the detached one's. Booting it second would let this assertion run while its own window still had + // time left, which would prove nothing about the rule. + const embedded = await bootDaemon({ ownership: 'embedded', idleTimeoutMs: 40 }); + const detached = await bootDaemon({ idleTimeoutMs: 40 }); + const detachedPaths = resolveDaemonPaths(detached.stateDir); + try { + // Idle reap releases the state directory it reaped, so that is the fact to wait for. By then the + // embedded runtime has been idle at least as long, so a window that ended one daemon's life is + // proof enough that it did not end the other's. + await vi.waitFor(() => expect(fs.existsSync(detachedPaths.infoPath)).toBe(false), { + timeout: 5_000, + }); + // A gap between requests is not the owner asking to be shut down. + expect((await embedded.sessionList()).ok).toBe(true); + } finally { + await detached.runtime.close(); + await embedded.runtime.close(); + } +}); + +test('an embedded runtime refuses a quiesce and remains serving until direct host close', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + const paths = resolveDaemonPaths(daemon.stateDir); + expect(JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')).ownership).toBe('embedded'); + expect(reasonOf(await daemon.quiesce())).toBe('daemon-owner-embedded'); + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')); + const shutdown = await sendSocketRequest(info.port, { + ...buildDaemonLifecycleRequest({ + mode: 'quiesce', + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime }, + requestId: 'not-exposed', + }), + command: 'daemon_shutdown', + }); + expect(shutdown.ok).toBe(false); + expect((await daemon.sessionList()).ok).toBe(true); + await daemon.runtime.close(); + expect(fs.existsSync(paths.infoPath)).toBe(false); +}); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index aa869d6721..11d4d929db 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -12,6 +12,7 @@ import { DAEMON_CAPABILITIES, resolveDaemonPaths, resolveDaemonServerMode, + type DaemonRuntimeOwnership, } from '@agent-device/daemon-contracts/daemon-resolution'; import { trackDownloadableArtifact } from '../../resources/artifact-tracking.ts'; import { @@ -55,7 +56,7 @@ import type { DaemonInvokeFn } from '../../core/daemon-request.ts'; import type { SessionRef } from '../../core/session-state.ts'; import { createDaemonIdleReap, - createDaemonQuiesce, + createDaemonLifecycleAdmission, daemonShuttingDownResponse, } from '../../sessions/server/daemon-idle-reap.ts'; import { withDaemonDiagnosticsScope } from '@agent-device/daemon-contracts/daemon-diagnostics-scope'; @@ -218,6 +219,8 @@ export type DaemonRuntimeOptions = { composeDeps: ComposeDaemonRuntimeDeps; /** Shutdown time budgets, for hosts and tests that need other than the defaults. */ shutdownBudgets?: { requestDrainMs?: number; pluginMs?: number; ownerMs?: number }; + /** The process owner. An embedded runtime ends through its host's direct close. */ + ownership?: DaemonRuntimeOwnership; }; export type DaemonRuntimeController = { @@ -511,6 +514,7 @@ async function startDaemonRuntimeOnHost( // well after this closure captures it. let shuttingDown = false; const inFlightRequests = new Set>(); + const ownership = options.ownership ?? 'detached'; const idleReap = createDaemonIdleReap({ sessionStore, getInFlightRequestCount: () => inFlightRequests.size, @@ -519,6 +523,7 @@ async function startDaemonRuntimeOnHost( void close(); }, env, + ownership, }); rollback.onRollback(() => { @@ -531,7 +536,8 @@ async function startDaemonRuntimeOnHost( // count it reads needs no allowance for it. The servers listen and `daemon.json` is published // before `shutdown` exists, and a daemon answers `draining` only once it has a drain path to run. let startingUp = true; - const quiesce = createDaemonQuiesce({ + const quiesce = createDaemonLifecycleAdmission({ + ownership: () => ownership, token, identity: { pid: daemonIdentity.pid, processStartTime: daemonIdentity.startTime }, sessionStore, @@ -572,6 +578,7 @@ async function startDaemonRuntimeOnHost( policyDigest: daemonPolicy?.digest, capabilities: DAEMON_CAPABILITIES, plugins: deps.pluginInventory, + ownership, }); if (socketPort) stdout.write(`AGENT_DEVICE_DAEMON_PORT=${socketPort}\n`); if (httpPort) stdout.write(`AGENT_DEVICE_DAEMON_HTTP_PORT=${httpPort}\n`); diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts b/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts index e663a012d8..e7b0b83e25 100644 --- a/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts +++ b/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts @@ -7,7 +7,7 @@ import { SessionStore } from '../../core/session-store.ts'; import type { SessionState } from '../../core/session-state.ts'; import { createDaemonIdleReap, - createDaemonQuiesce, + createDaemonLifecycleAdmission, hasActiveRecording, hasReapBlockingOpenSessions, isDaemonIdle, @@ -53,6 +53,18 @@ test('resolveDaemonIdleReapMs ignores invalid overrides', () => { assert.equal(resolveDaemonIdleReapMs({ AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '-5' }), 5 * 60_000); }); +test('resolveDaemonIdleReapMs gives a host-owned runtime no window', () => { + // The env window reaches a detached daemon only. A host-owned runtime is ended by its host, and a + // build that cannot place a record's owner never reaches this call: a runtime states its own + // ownership when it starts, so "unrecognized" is a reader's verdict about daemon.json, not a state + // the owner of a running daemon can be in. + assert.equal( + resolveDaemonIdleReapMs({ AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '1200' }, 'embedded'), + 0, + ); + assert.equal(resolveDaemonIdleReapMs({}, 'embedded'), 0); +}); + test('hasReapBlockingOpenSessions reflects the session store', () => { assert.equal(hasReapBlockingOpenSessions(sessionStore), false); sessionStore.publish('default', makeSession()); @@ -294,9 +306,32 @@ test('cancel prevents a scheduled reap from firing', async () => { assert.equal(reaped, 0); }); +test("a host that owns the runtime's lifetime never has it reaped on a timer", async () => { + vi.useFakeTimers(); + let reaped = 0; + const idleReap = createDaemonIdleReap({ + sessionStore, + getInFlightRequestCount: () => 0, + hasRetainedLeases: () => false, + onIdleReap: () => { + reaped++; + }, + // The env window a detached daemon would use, overridden by the owner's rule. + env: { AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '40' }, + ownership: 'embedded', + }); + + idleReap.noteActivity(); + await vi.advanceTimersByTimeAsync(1_000); + + assert.equal(reaped, 0); + assert.equal(idleReap.idleMs, 0); +}); + test('a daemon that cannot prove its start time refuses every quiesce', async () => { const drain = vi.fn(); - const quiesce = createDaemonQuiesce({ + const quiesce = createDaemonLifecycleAdmission({ + ownership: () => 'detached', token: 'daemon-token', identity: { pid: process.pid, processStartTime: null }, sessionStore, diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap.ts b/packages/daemon/src/sessions/server/daemon-idle-reap.ts index c65cdd2fee..db102c2858 100644 --- a/packages/daemon/src/sessions/server/daemon-idle-reap.ts +++ b/packages/daemon/src/sessions/server/daemon-idle-reap.ts @@ -1,17 +1,19 @@ import { - DAEMON_QUIESCE_COMMAND, - DAEMON_QUIESCE_REASONS, + DAEMON_LIFECYCLE_REASONS, DAEMON_SHUTTING_DOWN_REASON, - readDaemonQuiesceFence, - type DaemonQuiesceCensus, - type DaemonQuiesceFence, + readDaemonLifecycleMode, + readDaemonLifecycleFence, + type DaemonLifecycleCensus, + type DaemonLifecycleFence, + type DaemonLifecycleMode, + type DaemonRuntimeOwnership, } from '@agent-device/daemon-contracts/daemon-resolution'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { timingSafeStringEqual } from '@agent-device/host-kit/transport'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; import type { LeaseRegistry } from '../../resources/lease-registry.ts'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SessionStore } from '../../core/session-store.ts'; import type { SessionState } from '../../core/session-state.ts'; import { isUncommittedRepairSession } from '../../core/session-replay-transaction.ts'; @@ -31,7 +33,14 @@ const DAEMON_IDLE_REAP_DEFAULT_MS = 5 * 60_000; // AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS overrides the window; 0 disables idle // reap and restores the pre-existing "runs until killed" behavior. -export function resolveDaemonIdleReapMs(env: NodeJS.ProcessEnv = process.env): number { +// A timer exit is something only a detached daemon does: a runtime whose host owns it is ended by +// that host, and one whose owner cannot be proven is treated the same way, because nothing here can +// show the window's owner is nobody. +export function resolveDaemonIdleReapMs( + env: NodeJS.ProcessEnv = process.env, + ownership: DaemonRuntimeOwnership = 'detached', +): number { + if (ownership !== 'detached') return 0; const raw = env.AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS?.trim(); if (raw) { const parsed = Number(raw); @@ -100,8 +109,15 @@ export function createDaemonIdleReap(params: { hasRetainedLeases: () => boolean; onIdleReap: () => void; env?: NodeJS.ProcessEnv; + /** + * Who owns the process this timer runs in. A host that started the runtime in its own process owns + * its lifetime: an idle window tuned for a detached daemon nobody is watching would take the runtime + * out from under that owner, so an embedded runtime is never reaped on a timer. Defaults to a + * detached daemon. + */ + ownership?: DaemonRuntimeOwnership; }): DaemonIdleReapController { - const idleMs = resolveDaemonIdleReapMs(params.env); + const idleMs = resolveDaemonIdleReapMs(params.env, params.ownership); let timer: NodeJS.Timeout | undefined; const isIdleNow = (): boolean => @@ -143,16 +159,12 @@ export function createDaemonIdleReap(params: { return { noteActivity, cancel, idleMs }; } -/** - * ADR 0006: a daemon leaves for a newer one only when it holds nothing, and it decides that - * itself. A client that read "idle" and then signaled would race every other client of the state - * directory; the daemon is the one place the answer and the closing of admission can be one step. - */ -export type DaemonQuiesce = Readonly<{ +/** Answers a fenced quiesce before dispatch and closes admission atomically. */ +export type DaemonLifecycleAdmission = Readonly<{ /** - * Puts the quiesce request and the closed admission it leaves behind in front of `handler`: a - * quiesce request gets its verdict, and any other request gets the shutting-down refusal once a - * quiesce has closed admission. + * Puts the lifecycle request and the closed admission it leaves behind in front of `handler`: a + * lifecycle request gets its verdict, and any other request gets the shutting-down refusal once a + * lifecycle request has closed admission. */ admit(handler: DaemonInvokeFn): DaemonInvokeFn; }>; @@ -168,22 +180,27 @@ export function daemonShuttingDownResponse(): DaemonResponse { }; } -export function createDaemonQuiesce(params: { +export function createDaemonLifecycleAdmission(params: { token: string; - identity: DaemonQuiesceFence; + identity: DaemonLifecycleFence; sessionStore: Pick; leaseRegistry: Pick; - /** Requests being served, not counting the quiesce request itself. */ + /** Requests being served, not counting the lifecycle request itself. */ inFlightRequestCount: () => number; /** Whether the daemon has yet to build the drain path `drain` runs. */ startingUp: () => boolean; - /** Starts the drain; called at most once, after the response is on its way. */ + /** Starts the drain once, after sending the response. */ drain: () => void; -}): DaemonQuiesce { + /** A host-owned runtime refuses a quiesce. */ + ownership: () => DaemonRuntimeOwnership; +}): DaemonLifecycleAdmission { let admitting = true; - const refuse = (error: AppError): DaemonResponse => ({ ok: false, error: normalizeError(error) }); + const refuse = (error: AppError): DaemonResponse => ({ + ok: false, + error: normalizeError(error), + }); - const quiesce = (req: DaemonRequest): DaemonResponse => { + const leave = (req: DaemonRequest, mode: DaemonLifecycleMode): DaemonResponse => { if (typeof req.token !== 'string' || !timingSafeStringEqual(req.token, params.token)) { return refuse(new AppError('UNAUTHORIZED', 'Invalid daemon token.')); } @@ -191,10 +208,10 @@ export function createDaemonQuiesce(params: { // host that owns the state directory may retire the daemon on it. if (req.internal?.publicNetworkOnly) { return refuse( - new AppError('UNAUTHORIZED', 'Daemon quiesce is not available to remote clients.'), + new AppError('UNAUTHORIZED', `Daemon ${mode} is not available to remote clients.`), ); } - const fence = readDaemonQuiesceFence(req.input); + const fence = readDaemonLifecycleFence(req.input); // A pid alone can name a recycled process, so a daemon that cannot prove its start time // cannot be addressed at all. if (!fence || fence.pid !== params.identity.pid) return refuse(instanceMismatch()); @@ -202,25 +219,26 @@ export function createDaemonQuiesce(params: { if (fence.processStartTime !== params.identity.processStartTime) { return refuse(instanceMismatch()); } + // Each mode belongs to one owner: a host ends an embedded runtime it started, and the process that + // started a detached daemon ends it by signalling its pid. Neither asks a stranger. + if (mode === 'quiesce' && params.ownership() === 'embedded') return refuse(ownerEmbedded()); // Admission closes and the census is read in the same synchronous step, so no request can be // admitted between the two; a refusal reopens admission in that same step. if (!admitting) return { ok: true, data: { outcome: 'draining' } }; admitting = false; const census = readCensus(); - if (isHeld(census)) { + // A daemon that is still starting has no drain path yet, so it accepts nothing, and a quiesce + // that finds work in hand refuses too: replacing that daemon would end another client's session. + const held = isHeld(census); + if (held) { admitting = true; - return refuse( - new AppError('COMMAND_FAILED', 'Daemon is busy and was not asked to leave.', { - reason: DAEMON_QUIESCE_REASONS.busy, - ...census, - }), - ); + return refuse(busy(census)); } - setImmediate(params.drain); + setImmediate(() => params.drain()); return { ok: true, data: { outcome: 'draining' } }; }; - const readCensus = (): DaemonQuiesceCensus => { + const readCensus = (): DaemonLifecycleCensus => { const sessions = params.sessionStore.toArray(); const repairSessions = sessions.filter(isUncommittedRepairSession).length; return { @@ -234,13 +252,14 @@ export function createDaemonQuiesce(params: { return { admit: (handler) => async (req) => { - if (req.command === DAEMON_QUIESCE_COMMAND) return quiesce(req); + const mode = readDaemonLifecycleMode(req.command); + if (mode) return leave(req, mode); return admitting ? await handler(req) : daemonShuttingDownResponse(); }, }; } -function isHeld(census: DaemonQuiesceCensus): boolean { +function isHeld(census: DaemonLifecycleCensus): boolean { return ( census.startingUp || census.inFlightRequests > 0 || @@ -250,8 +269,23 @@ function isHeld(census: DaemonQuiesceCensus): boolean { ); } +function busy(census: DaemonLifecycleCensus): AppError { + return new AppError('COMMAND_FAILED', 'Daemon is busy and was not asked to leave.', { + reason: DAEMON_LIFECYCLE_REASONS.busy, + ...census, + }); +} + function instanceMismatch(): AppError { - return new AppError('COMMAND_FAILED', 'Quiesce request names a different daemon instance.', { - reason: DAEMON_QUIESCE_REASONS.instanceMismatch, + return new AppError('COMMAND_FAILED', 'A lifecycle request names a different daemon instance.', { + reason: DAEMON_LIFECYCLE_REASONS.instanceMismatch, }); } + +function ownerEmbedded(): AppError { + return new AppError( + 'COMMAND_FAILED', + 'A host process owns this daemon, so only that host can retire it.', + { reason: DAEMON_LIFECYCLE_REASONS.ownerEmbedded }, + ); +} diff --git a/scripts/clean-daemon.ts b/scripts/clean-daemon.ts index 8b7b10cdaa..0cc0d986b1 100644 --- a/scripts/clean-daemon.ts +++ b/scripts/clean-daemon.ts @@ -5,7 +5,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { withCommandHost } from '@agent-device/host-kit/command'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -import { readRegisteredDaemonIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; import { stopAndRetireDaemon, recoverAbandonedDaemonRegistration, @@ -22,7 +22,7 @@ const paths = resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR, { }); const retirement = await stopAndRetireDaemon({ paths, - observed: readRegisteredDaemonIdentity(paths.infoPath), + observed: readCleanupIdentity(paths.infoPath), mode: 'graceful', termTimeoutMs: DAEMON_TERM_TIMEOUT_MS, killTimeoutMs: DAEMON_KILL_TIMEOUT_MS, @@ -65,7 +65,7 @@ async function pruneStaleDevStateDirs(): Promise { const paths = resolveDaemonPaths(dirPath, { cwd: process.cwd() }); const result = await recoverAbandonedDaemonRegistration({ paths, - observed: readRegisteredDaemonIdentity(paths.infoPath), + observed: readCleanupIdentity(paths.infoPath), }); if (result.status === 'retired') process.stdout.write( @@ -107,3 +107,10 @@ function statMtimeMs(filePath: string): number { return 0; } } + +function readCleanupIdentity(infoPath: string) { + const { identity, ownership } = readDaemonStopIdentity(infoPath); + return identity && ownership !== 'absent' && ownership !== 'unreadable' + ? { ...identity, ownership } + : null; +} diff --git a/src/__tests__/daemon-registration-embedded-owner.test.ts b/src/__tests__/daemon-registration-embedded-owner.test.ts new file mode 100644 index 0000000000..76b39764b3 --- /dev/null +++ b/src/__tests__/daemon-registration-embedded-owner.test.ts @@ -0,0 +1,227 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { readCurrentOwnerIdentity, isProcessAlive } from '@agent-device/host-kit/process'; +import { stopAndRetireDaemon } from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { stopRecordedDaemon } from '../daemon-client/daemon-client-stop.ts'; +import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; + +// What the registration owner may do to a record a host process owns: prove the lifetime is gone +// before removing anything, and never signal the process the record names. + +/** + * Whether the signal path was reached, and with what. A signal needs the observed pid to pass the + * daemon-command-line check, and the process these rows name is the vitest worker, which fails that + * check. Stubbed to accept it, the rows ask the question they mean to ask: does the record's owner keep + * the stop away from the signal path, or only a process-table lookup that happens to refuse? + */ +const signalPath = vi.hoisted(() => ({ attempted: [] as number[] })); + +vi.mock('@agent-device/daemon-contracts/daemon-process', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + stopDaemonProcess: async ( + observed: Parameters[0], + _options: Parameters[1], + ) => { + // Records the attempt instead of signaling, so the row can tell a refused stop from one that + // signaled its own test worker. An exit that cannot be confirmed is what a real signal that did + // not kill anything answers with. + signalPath.attempted.push(observed.pid); + return { status: 'retained' as const, reason: 'exit-timeout' as const }; + }, + }; +}); + +const ownIdentity = readCurrentOwnerIdentity(); +const deadIdentity = { pid: 999_999_999, startTime: 'dead-start' }; + +/** What a reader that read `ownership` from the record passes on, which is what retirement judges. */ +function observedAs( + identity: { pid: number; startTime: string | null }, + ownership: 'embedded' | 'unknown' | 'detached', +) { + return { ...identity, ownership }; +} + +afterEach(() => { + vi.restoreAllMocks(); + signalPath.attempted.length = 0; +}); + +/** Writes a record that claims `ownership` for a live process, unreadable variants included. */ +function writeRecord( + paths: ReturnType, + identity: unknown, + ownership: unknown = 'embedded', +) { + const { pid, startTime } = identity as { pid: number; startTime: string }; + fs.writeFileSync( + paths.infoPath, + JSON.stringify({ + pid, + processStartTime: startTime, + token: 'token', + port: 4210, + ...(ownership === 'corrupt' ? {} : { ownership }), + }), + ); + if (ownership === 'corrupt') fs.writeFileSync(paths.infoPath, '{not json'); +} + +function writeEmbeddedRecord(paths: ReturnType, identity: unknown) { + writeRecord(paths, identity); +} + +// A host owns an embedded runtime and ends it with its own close, so retiring that record from outside +// would take the state directory out from under a process that is still serving. +test('retirement of an embedded record proves the lifetime is gone and signals nothing', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-embedded-retirement-')); + writeEmbeddedRecord(paths, ownIdentity); + const before = fs.readFileSync(paths.infoPath, 'utf8'); + const signals = vi.spyOn(process, 'kill'); + + for (const mode of ['graceful', 'force'] as const) { + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(ownIdentity, 'embedded'), + mode, + lockTimeoutMs: 0, + }); + assert.equal(result.status, 'retained', JSON.stringify(result)); + if (result.status !== 'retained') assert.fail('an embedded owner was retired'); + assert.equal(result.reason, 'embedded-owner'); + assert.equal(result.termination?.status, 'retained'); + } + + await assert.rejects( + stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), { + paths, + host: createTestHostEnvironment(), + }), + (error: unknown) => + error instanceof AppError && + error.details?.reason === 'daemon-owner-embedded' && + /Close the host/.test(String(error.details.hint)), + ); + + // The row that gives this file its teeth: the stub above accepts this pid for the signal path, so + // only the record's owner stands between this stop and a SIGTERM to the test worker itself. + assert.deepEqual(signalPath.attempted, [], 'an embedded owner is never reached for a signal'); + assert.deepEqual( + signals.mock.calls.filter(([, requested]) => requested !== 0), + [], + 'retirement must not reach into a host process', + ); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before); + assert.equal(isProcessAlive(ownIdentity.pid), true); +}); + +// The control that keeps the row above honest: the same live pid, the same stub, and a record proving +// the process is a daemon of its own, and the signal path is taken. Without this row the embedded +// refusal could still be a process-table lookup refusing the worker rather than the owner's rule. +test('a detached record for the same live lifetime does take the signal path', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-detached-signal-')); + writeRecord(paths, ownIdentity, 'detached'); + + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(ownIdentity, 'detached'), + mode: 'graceful', + lockTimeoutMs: 0, + }); + + assert.deepEqual( + signalPath.attempted, + [ownIdentity.pid], + 'a proven detached daemon is the one record a stop signals', + ); + assert.equal(result.status, 'retained', JSON.stringify(result)); +}); + +test('an embedded record whose lifetime is provably gone retires like any other abandoned one', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-embedded-dead-')); + writeEmbeddedRecord(paths, deadIdentity); + + // A host that died without closing left the record behind; keeping it would wedge the state directory + // for every later client with nothing left to close. + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(deadIdentity, 'embedded'), + mode: 'graceful', + lockTimeoutMs: 0, + }); + assert.equal(result.status, 'retired', JSON.stringify(result)); + assert.equal(result.removedInfo, true); + assert.equal(fs.existsSync(paths.infoPath), false); +}); + +// A record this build cannot read is not a licence to signal what it names: `detached` is the one +// ownership that authorizes a signal, and an unreadable or unrecognized record proves nothing about +// who owns the process. This is the fail-closed side of the same rule the embedded row above proves. +test('retirement refuses to signal a record whose owner it cannot recognize', async () => { + for (const ownership of ['sideways', 'corrupt'] as const) { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-owner-unknown-')); + writeRecord(paths, ownIdentity, ownership); + const before = fs.readFileSync(paths.infoPath, 'utf8'); + const signals = vi.spyOn(process, 'kill'); + + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(ownIdentity, 'unknown'), + mode: 'graceful', + lockTimeoutMs: 0, + }); + assert.equal(result.status, 'retained', ownership); + if (result.status !== 'retained') + assert.fail(`${ownership}: an unrecognized owner was retired`); + // The reason differs from the embedded one: nothing here proves the pid is a daemon of its own, so + // an operator fixes this by inspecting or updating the record, not by closing a host. + assert.equal(result.reason, 'owner-unproven', ownership); + if (ownership === 'sideways') + await assert.rejects( + stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), { + paths, + host: createTestHostEnvironment(), + }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'daemon-owner-unrecognized', + ); + assert.deepEqual( + signalPath.attempted, + [], + `${ownership}: an unrecognized owner is never reached for a signal`, + ); + assert.deepEqual( + signals.mock.calls.filter(([, requested]) => requested !== 0), + [], + `${ownership}: an unrecognized owner is never signaled`, + ); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before, ownership); + vi.mocked(process.kill).mockClear(); + } +}); + +test.each([ownIdentity, deadIdentity])( + 'a JS caller without ownership only proves exit: $pid', + async (identity) => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-owner-omitted-')); + writeRecord(paths, identity, 'detached'); + const result = await stopAndRetireDaemon({ + paths, + // @ts-expect-error JavaScript/strip-types callers can bypass the required ownership field. + observed: identity, + mode: 'force', + lockTimeoutMs: 0, + }); + assert.equal(result.status, identity === ownIdentity ? 'retained' : 'retired'); + if (result.status === 'retained') assert.equal(result.reason, 'owner-unproven'); + assert.deepEqual(signalPath.attempted, []); + assert.equal(fs.existsSync(paths.infoPath), identity === ownIdentity); + }, +); diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index 8f86229aa3..05f9ca9440 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -16,8 +16,12 @@ import { import { resolveDaemonPaths, type DaemonPaths, + type ObservedDaemonIdentity, } from '@agent-device/daemon-contracts/daemon-resolution'; -import { readRegisteredDaemonOwnership } from '@agent-device/daemon-contracts/daemon-registration'; +import { + readDaemonStopIdentity, + readRegisteredDaemonOwnership, +} from '@agent-device/daemon-contracts/daemon-registration'; import { readDaemonShutdownReport } from '@agent-device/daemon-contracts/daemon-shutdown-report'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { @@ -44,7 +48,26 @@ const fields = { codeOrigin: 'checkout' as const, codeSignature: 'signature', }; -const ownIdentity = readCurrentOwnerIdentity(); +const ownIdentity = observedOf(readCurrentOwnerIdentity()); + +/** + * Every record this file publishes describes a detached daemon: these rows write the file directly and + * state no owner, which is exactly what a record predating the ownership field states. + */ +function observedOf( + identity: T, +): ObservedDaemonIdentity { + return { ...identity, ownership: 'detached' as const }; +} + +/** The same observation a stop routes on: the record this file published, read once. */ +function readStopObserved(infoPath: string): ObservedDaemonIdentity { + const record = readDaemonStopIdentity(infoPath); + if (!record.identity || record.ownership !== 'detached') { + throw new Error(`the fixture published a ${record.ownership} record`); + } + return observedOf(record.identity); +} const report = { providerReleases: { released: [], pending: [] }, claims: { released: [], orphaned: [], superseded: [], unattributable: [] }, @@ -314,11 +337,11 @@ test('unlink disappearance is settled, other failures remain primary even when r assert.match(fs.readFileSync(second.paths.logPath, 'utf8'), /daemon_registration_release_failed/); }); -const deadIdentity = { pid: 999_999_999, startTime: 'dead-start' }; +const deadIdentity = observedOf({ pid: 999_999_999, startTime: 'dead-start' }); for (const scenario of [ { name: 'missing-start', - observed: { pid: process.pid, startTime: null }, + observed: observedOf({ pid: process.pid, startTime: null }), published: ownIdentity, retired: false, }, @@ -362,13 +385,17 @@ test('abandoned recovery never signals a live owner and absence is inspected und }); }); +// A host owns an embedded runtime and ends it with its own close, so retiring that record from outside +// would take the state directory out from under a process that is still serving. The rows that prove it +// are in daemon-registration-embedded-owner.test.ts, next to the owner rule they exercise. + test('retirement preserves a winner holding the startup lock instead of stopping or unlinking it', async () => { const { paths, owner } = await acquire(); owner.publish({ ...fields, token: 'winning-token' }); const before = fs.readFileSync(paths.infoPath, 'utf8'); const result = await stopAndRetireDaemon({ paths, - observed: { pid: 999_999_999, startTime: 'dead-start' }, + observed: observedOf({ pid: 999_999_999, startTime: 'dead-start' }), mode: 'force', lockTimeoutMs: 0, }); @@ -384,7 +411,7 @@ test('retirement preserves a winner holding the startup lock instead of stopping test('metadata failure remains primary and normalized through a secondary release failure', async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-failures-')); - const observed = { pid: 999_999_999, startTime: 'dead-start' }; + const observed = observedOf({ pid: 999_999_999, startTime: 'dead-start' }); replaceInfo(paths, observed.pid, observed.startTime); const primary = Object.assign(new Error('metadata failure'), { code: 'EIO' }); const originalUnlink = fs.unlinkSync; @@ -408,7 +435,7 @@ test('metadata failure remains primary and normalized through a secondary releas test('release failure reports partial retirement rather than success', async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-partial-')); - const observed = { pid: 999_999_999, startTime: 'dead-start' }; + const observed = observedOf({ pid: 999_999_999, startTime: 'dead-start' }); replaceInfo(paths, observed.pid, observed.startTime); const original = fs.rmdirSync; vi.spyOn(fs, 'rmdirSync').mockImplementation((target, options) => { @@ -469,7 +496,7 @@ test('private retirement closes startup admission, joins the actual child and ne assert.equal((await contender.exited).exitCode, DAEMON_STARTUP_EXIT_CODES.busy); const input = { paths, - observed: { pid: launch.pid, startTime: launch.startTime }, + observed: observedOf({ pid: launch.pid, startTime: launch.startTime }), mode: 'graceful' as const, ownedStateDir, }; @@ -515,7 +542,7 @@ test('private retirement retains the directory while an earlier actual startup c await waitForFixtureFile(paths.infoPath); const result = await stopAndRetireDaemon({ paths, - observed: { pid: second.pid, startTime: second.startTime ?? null }, + observed: observedOf({ pid: second.pid, startTime: second.startTime ?? null }), mode: 'graceful', ownedStateDir, startupJoinTimeoutMs: 0, @@ -581,7 +608,7 @@ for (const [failure, reason, expectedSignals] of privateStartupFailures) { }); const pending = stopAndRetireDaemon({ paths, - observed: { pid: launch.pid, startTime: launch.startTime ?? null }, + observed: observedOf({ pid: launch.pid, startTime: launch.startTime ?? null }), mode: 'graceful', ownedStateDir, startupJoinTimeoutMs: 500, @@ -657,7 +684,7 @@ test('private startup completion stays bounded without published birth proof and const signals = vi.spyOn(process, 'kill'); const pending = stopAndRetireDaemon({ paths, - observed: { pid: launch.pid, startTime: null }, + observed: observedOf({ pid: launch.pid, startTime: null }), mode: 'force', ownedStateDir, startupJoinTimeoutMs: 0, @@ -719,7 +746,7 @@ for (const contents of [ await waitForFixtureFile(paths.infoPath); const result = await stopAndRetireDaemon({ paths, - observed: { pid: launch.pid, startTime: launch.startTime ?? null }, + observed: observedOf({ pid: launch.pid, startTime: launch.startTime ?? null }), mode: 'graceful', ownedStateDir, }); @@ -887,7 +914,14 @@ for (const mode of ['graceful', 'forced'] as const) { const child = spawnRegisteredDaemonFixture(paths, fields, undefined); try { await waitForFixtureFile(paths.infoPath); - const result = await stopDaemon({ paths, graceTimeoutMs: 30, killTimeoutMs: 1_000 }); + const result = await stopDaemon({ + paths, + // The row routes the stop the way the CLI does: one read of the record the fixture published, + // and its lifetime and owner handed to the funnel unchanged. + observed: readStopObserved(paths.infoPath), + graceTimeoutMs: 30, + killTimeoutMs: 1_000, + }); assert.equal(result.stopped, true); assert.equal(result.mode, mode); assert.equal(result.cleanupConfidence, mode === 'forced' ? 'unknown' : 'known'); diff --git a/src/cli/commands/__tests__/daemon.test.ts b/src/cli/commands/__tests__/daemon.test.ts index 580714bf22..8536fe09db 100644 --- a/src/cli/commands/__tests__/daemon.test.ts +++ b/src/cli/commands/__tests__/daemon.test.ts @@ -12,7 +12,6 @@ const mocks = vi.hoisted(() => ({ cleanupDaemonOwner: vi.fn(async () => undefined), readDaemonShutdownReport: vi.fn(), readDaemonStatus: vi.fn(), - readDaemonStopIdentity: vi.fn(), stopDaemon: vi.fn(), writeCommandOutput: vi.fn(), })); @@ -20,8 +19,8 @@ const mocks = vi.hoisted(() => ({ // The mock specifier must be the SUT's exact `@agent-device/daemon-contracts/daemon-stop`: the // mutation lane aliases only the manifest's exact entries, so the `./testing/*` pattern // form resolved outside the Stryker sandbox and the mock silently missed (#3546 CI). -vi.mock('@agent-device/daemon-contracts/daemon-stop', () => ({ - readDaemonStopIdentity: mocks.readDaemonStopIdentity, +vi.mock('@agent-device/daemon-contracts/daemon-stop', async (importOriginal) => ({ + ...(await importOriginal()), stopDaemon: mocks.stopDaemon, })); vi.mock('@agent-device/daemon-contracts/daemon-shutdown-report', () => ({ @@ -30,6 +29,8 @@ vi.mock('@agent-device/daemon-contracts/daemon-shutdown-report', () => ({ vi.mock('../../../platform-runtime-daemon-owner-cleanup.ts', () => ({ createDaemonOwnerCleanup: () => ({ cleanup: mocks.cleanupDaemonOwner }), })); +// `readDaemonStopIdentity` stays real: these tests write the record they want judged, which is the +// only way to show routing, retirement, and a `--clean` walk judge one read of one file. vi.mock('@agent-device/daemon-contracts/daemon-registration', async (importOriginal) => ({ ...(await importOriginal()), readDaemonStatus: mocks.readDaemonStatus, @@ -37,6 +38,7 @@ vi.mock('@agent-device/daemon-contracts/daemon-registration', async (importOrigi vi.mock('../shared.ts', () => ({ writeCommandOutput: mocks.writeCommandOutput })); import { daemonCommand } from '../daemon.ts'; +import * as daemonClientStop from '../../../daemon-client/daemon-client-stop.ts'; bindTestCommandHost(); @@ -52,6 +54,17 @@ const GRACEFUL_RESULT: DaemonStopResult = { warnings: [], }; +/** A detached daemon's record, which is the shape a `--clean` cleanup is allowed to follow. */ +function writeDetachedRecord(stateDir: string): void { + fs.writeFileSync( + `${stateDir}/daemon.json`, + JSON.stringify({ pid: 123, processStartTime: 'start' }), + { + mode: 0o600, + }, + ); +} + afterEach(() => { vi.clearAllMocks(); }); @@ -81,8 +94,14 @@ test('accepts only daemon stop and status', async () => { test('merges a graceful shutdown report and cleans runner leases with the start-time identity', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue({ pid: 123, processStartTime: 'start-time' }); - mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); + writeDetachedRecord(stateDir); + mocks.stopDaemon.mockImplementationOnce(async () => { + fs.writeFileSync( + `${stateDir}/daemon.json`, + JSON.stringify({ pid: 456, processStartTime: 'successor', ownership: 'embedded' }), + ); + return GRACEFUL_RESULT; + }); const claim = { deviceKey: 'local:android:none:emulator-5554', session: 'default', @@ -105,9 +124,15 @@ test('merges a graceful shutdown report and cleans runner leases with the start- client: {} as never, }); + // The lifetime handed to the retirement is the one read out of that record, with the owner the + // same read stated: routing and retirement judge one observation. + expect(mocks.stopDaemon).toHaveBeenCalledWith({ + paths: expect.objectContaining({ baseDir: stateDir }), + observed: { pid: 123, startTime: 'start', ownership: 'detached' }, + }); expect(mocks.cleanupDaemonOwner).toHaveBeenCalledWith({ pid: 123, - startTime: 'start-time', + startTime: 'start', }); expect(mocks.writeCommandOutput).toHaveBeenCalledWith( expect.objectContaining({ clean: true, json: false }), @@ -133,7 +158,6 @@ test('merges a graceful shutdown report and cleans runner leases with the start- test('reports graceful provider cleanup as unknown when the shutdown report is unavailable', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue(null); mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); mocks.readDaemonShutdownReport.mockReturnValue(null); @@ -145,6 +169,12 @@ test('reports graceful provider cleanup as unknown when the shutdown report is u client: {} as never, }); + // No record at all is not a refusal: nothing claims the state directory, so this stop hands the + // retirement no lifetime and still reports what it was told. + expect(mocks.stopDaemon).toHaveBeenCalledWith({ + paths: expect.objectContaining({ baseDir: stateDir }), + observed: null, + }); expect(mocks.writeCommandOutput).toHaveBeenCalledWith( expect.objectContaining({ json: false }), expect.objectContaining({ @@ -162,7 +192,7 @@ test('reports graceful provider cleanup as unknown when the shutdown report is u test('warns in text output when a graceful stop leaves an orphaned claim', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue({ pid: 123, processStartTime: 'start-time' }); + writeDetachedRecord(stateDir); mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); const claim = { deviceKey: 'local:android:none:emulator-5554', @@ -200,7 +230,7 @@ test('warns in text output when a graceful stop leaves an orphaned claim', async test('routes an unattributable claim to the default status view, never to --stale', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue({ pid: 123, processStartTime: 'start-time' }); + writeDetachedRecord(stateDir); mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); const claim = { deviceKey: 'local:android:none:emulator-5554', @@ -247,6 +277,7 @@ test('daemon status reads the daemon record without stopping or contacting the d state: 'running', stateDir, pid: 321, + ownership: 'embedded', version: '0.21.30', plugins: [ { @@ -285,7 +316,105 @@ test('daemon status reads the daemon record without stopping or contacting the d const render = mocks.writeCommandOutput.mock.calls[0]![2] as () => string; expect(render()).toContain('testmu [store] @agent-device/testmu@1.0.0 - failed: bad plugin'); expect(render()).toContain('browserstack [builtin] builtin - active'); + // A host-owned pid changes what a stop may do to it, so status says who owns it. + expect(render()).toContain('pid 321, host-owned'); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// `--clean` walks the owner's process tree. A host process's tree is that host's whole program, so the +// flag is refused for a host-owned record rather than reaching past the runtime's own close. +test.each([ + [ + 'embedded', + true, + JSON.stringify({ pid: 555, processStartTime: 'start', ownership: 'embedded' }), + 'daemon-clean-not-detached', + /Close the host/, + ], + [ + 'unknown', + true, + JSON.stringify({ pid: 555, processStartTime: 'start', ownership: 'sideways' }), + 'daemon-clean-not-detached', + /remove .*daemon\.json/, + ], + ['unreadable', true, '{not json', 'daemon-clean-not-detached', /remove .*daemon\.json/], + ['unreadable', false, '{not json', 'daemon-record-unreadable', /Inspect or remove that record/], +] as const)('daemon stop refuses %s with clean=%s', async (_label, clean, record, reason, hint) => { + const stateDir = mkdtempForTestSync('agent-device-daemon-command-refused-'); + fs.writeFileSync(`${stateDir}/daemon.json`, record, { mode: 0o600 }); + try { + await expect( + daemonCommand({ + host: createTestHostEnvironment(), + positionals: ['stop'], + flags: { clean, help: false, json: false, stateDir, version: false }, + client: {} as never, + }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason, stateDir, hint: expect.stringMatching(hint) }, + }); + expect(mocks.stopDaemon).not.toHaveBeenCalled(); + expect(mocks.cleanupDaemonOwner).not.toHaveBeenCalled(); + expect(fs.readFileSync(`${stateDir}/daemon.json`, 'utf8')).toBe(record); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// Status is the surface an operator reads before deciding what a stop may do, so an owner this build +// cannot place is said out loud instead of reading as an ordinary detached daemon. +test('daemon status names a record whose owner it cannot recognize', async () => { + mocks.readDaemonStatus.mockReturnValue({ + state: 'running', + stateDir: '/tmp/agent-device-daemon-status-unknown', + pid: 321, + ownership: 'unknown', + }); + + await daemonCommand({ + host: createTestHostEnvironment(), + positionals: ['status'], + flags: { help: false, json: false, stateDir: '/tmp/x', version: false }, + client: {} as never, + }); + + const render = mocks.writeCommandOutput.mock.calls.at(-1)![2] as () => string; + expect(render()).toContain('pid 321, owner-not-recognized'); +}); + +test('daemon stop of a host-owned record goes through the one router with the host the command runs on', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-command-embedded-'); + fs.writeFileSync( + `${stateDir}/daemon.json`, + JSON.stringify({ pid: 555, processStartTime: 'start-time', ownership: 'embedded' }), + { mode: 0o600 }, + ); + vi.spyOn(daemonClientStop, 'stopRecordedDaemon').mockResolvedValue({ + ...GRACEFUL_RESULT, + warnings: ['1 in-flight request ignored cancellation and was abandoned.'], + }); + + const host = createTestHostEnvironment(); + + try { + await daemonCommand({ + host, + positionals: ['stop'], + flags: { help: false, json: false, stateDir, version: false }, + client: {} as never, + }); + + // A host-owned runtime is never signaled, so it never reaches the retirement a detached stop uses. + expect(mocks.stopDaemon).not.toHaveBeenCalled(); + const render = mocks.writeCommandOutput.mock.calls[0]![2] as () => string; + expect(render()).toContain('Daemon stopped (graceful).'); + expect(render()).toContain('ignored cancellation'); } finally { + vi.restoreAllMocks(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts index f14624b46f..56818291a0 100644 --- a/src/cli/commands/daemon.ts +++ b/src/cli/commands/daemon.ts @@ -3,21 +3,22 @@ import { type PluginInventoryEntry, } from '@agent-device/contracts/plugin-inventory'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -import { - readDaemonStopIdentity, - stopDaemon, - type DaemonStopResult, -} from '@agent-device/daemon-contracts/daemon-stop'; +import type { DaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { isUnusableRecord, readDaemonStatus, + readDaemonStopIdentity, + readStopCleanupOwner, type DaemonStatus, + type DaemonStopIdentity, } from '@agent-device/daemon-contracts/daemon-registration'; -import { readDaemonShutdownReport } from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import type { DaemonStopResult } from '@agent-device/daemon-contracts/daemon-stop'; import { AppError } from '@agent-device/kernel/errors'; +import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell'; import { writeCommandOutput } from './shared.ts'; import type { ClientCommandHandler } from './router-types.ts'; import { createDaemonOwnerCleanup } from '../../platform-runtime-daemon-owner-cleanup.ts'; +import { stopRecordedDaemon } from '../../daemon-client/daemon-client-stop.ts'; export const daemonCommand: ClientCommandHandler = async ({ positionals, flags, host }) => { const daemonOwnerCleanup = createDaemonOwnerCleanup(host); @@ -35,15 +36,23 @@ export const daemonCommand: ClientCommandHandler = async ({ positionals, flags, await writeCommandOutput(flags, shown, () => renderDaemonStatus(shown)); return true; } - const identity = readDaemonStopIdentity(paths.infoPath); - const stopped = await stopDaemon({ paths }); - const report = stopped.mode === 'graceful' ? readDaemonShutdownReport(paths.baseDir) : null; - const result = mergeShutdownReport(stopped, report); - const shouldClean = flags.clean === true && identity !== null && result.stopped; + // One read decides every route below, so routing, retirement, and a `--clean` walk judge the same + // record rather than three snapshots of a file another process may be rewriting. + const record = readDaemonStopIdentity(paths.infoPath); + // Retained-runner cleanup works across the owner's whole process tree, which a host process shares + // with whatever else it runs. An embedded runtime's own close is the boundary a stop may cross, and + // a record that cannot prove its owner is not a licence to walk a process tree either. No record at all + // is not a refusal: there is nothing claimed, so there is nothing to clean. + if (flags.clean === true && record.ownership !== 'absent' && record.ownership !== 'detached') { + throw daemonCleanRefusedForUnprovenOwner(paths, record.ownership); + } + const result = await stopRecordedDaemon(record, { paths, host }); + const cleanupOwner = readStopCleanupOwner(record); + const shouldClean = flags.clean === true && cleanupOwner !== null && result.stopped; if (shouldClean) { await daemonOwnerCleanup.cleanup({ - pid: identity.pid, - startTime: identity.processStartTime, + pid: cleanupOwner.pid, + startTime: cleanupOwner.processStartTime, }); } const data = { ...result, clean: shouldClean }; @@ -51,82 +60,29 @@ export const daemonCommand: ClientCommandHandler = async ({ positionals, flags, return true; }; -function mergeShutdownReport( - stopped: DaemonStopResult, - report: ReturnType, -): DaemonStopResult { - if (stopped.mode !== 'graceful' || report) { - return report - ? { - ...stopped, - providerReleases: { status: 'completed', ...report.providerReleases }, - claimsReleased: report.claims.released, - claimsOrphaned: report.claims.orphaned, - claimsSuperseded: report.claims.superseded, - claimsUnattributable: report.claims.unattributable, - warnings: [ - ...stopped.warnings, - ...supersededClaimWarnings(report.claims.superseded), - ...orphanedClaimWarnings(report.claims.orphaned), - ...unattributableClaimWarnings(report.claims.unattributable), - ], - } - : stopped; - } - return { - ...stopped, - cleanupConfidence: 'unknown', - providerReleases: { status: 'unknown', released: [], pending: null }, - warnings: [ - ...stopped.warnings, - 'The graceful shutdown report is unavailable, so provider cleanup state is unknown. Provider allocations may remain active.', - ], - }; +/** Refuses PID-wide cleanup when the registration cannot prove a detached daemon. */ +function daemonCleanRefusedForUnprovenOwner( + paths: DaemonPaths, + record: DaemonStopIdentity['ownership'], +): AppError { + return new AppError( + 'COMMAND_FAILED', + 'This daemon record does not prove the process is a daemon of its own, so its retained-runner cleanup cannot be run from here.', + { + reason: 'daemon-clean-not-detached', + stateDir: paths.baseDir, + ownership: record, + hint: daemonCleanRefusalHint(paths, record), + }, + ); } -/** A superseded claim is not a failure to report as one, but the operator's - * device is now owned elsewhere, so it must not pass silently. */ -function supersededClaimWarnings(superseded: DaemonStopResult['claimsSuperseded']): string[] { - if (superseded.length === 0) return []; - const devices = superseded.map((claim) => claim.deviceId).join(', '); - return [ - `Another owner had already claimed ${devices} before this daemon released it, so those devices are now owned elsewhere.`, - ]; -} - -/** An orphaned claim keeps holding its device after the daemon is gone, and - * only `device release --stale` or the next open settles it — say so instead - * of leaving the block discoverable through --json alone. - * - * A record whose owner could not be attributed is NOT reported here. Those are what - * `--stale` hides and refuses, so this sentence would send the operator to a route that cannot - * succeed; {@link unattributableClaimWarnings} names the view that does show them. */ -function orphanedClaimWarnings(orphaned: DaemonStopResult['claimsOrphaned']): string[] { - if (orphaned.length === 0) return []; - const devices = orphaned.map((claim) => claim.deviceId).join(', '); - return [ - `Ownership of ${devices} was not released cleanly; the claim now blocks other owners until it is settled. Inspect with: agent-device device status --stale, then release with: agent-device device release --stale.`, - ]; -} - -/** - * What the three provable buckets each get an exact remedy for, this one does not: settling a claim - * always rests on a proof about its owner, and this record names none. So the warning states only - * what is known, points at the view that lists these records — the default one, because `--stale` - * filters records without a decodable owner out — and does not promise a release that would clear - * them. `device release --stale` refuses every record that names no owner, and the next `open` refuses - * to overwrite one. - */ -function unattributableClaimWarnings( - unattributable: DaemonStopResult['claimsUnattributable'], -): string[] { - if (unattributable.length === 0) return []; - const devices = unattributable.map((claim) => claim.deviceId).join(', '); - return [ - `A claim for ${devices} remains whose owner could not be read, so this daemon cannot say whether those devices are free. ` + - 'No device release route settles it: --stale proves staleness from the recorded owner and refuses records that name none, ' + - 'and open refuses to overwrite them. Inspect with: agent-device device status.', - ]; +function daemonCleanRefusalHint( + paths: DaemonPaths, + record: DaemonStopIdentity['ownership'], +): string { + if (record === 'embedded') return 'Close the host that started this runtime.'; + return `Nothing was signaled. Check what owns that process, then repair the record: stop it by its own means and remove ${shellQuoteIfNeeded(`${paths.baseDir}/daemon.json`)} so a daemon can register again.`; } function renderDaemonStop( @@ -147,7 +103,18 @@ function renderDaemonStatus( return `The daemon record for ${status.stateDir} is ${status.state === 'invalid' ? 'not a valid record' : 'unreadable'}, so whether a daemon runs there is unknown.`; } if (status.state === 'stopped') return `No running daemon for ${status.stateDir}.`; - const details = [`pid ${status.pid}`, status.version, status.transport].filter(Boolean); + const details = [ + `pid ${status.pid}`, + // An embedded pid belongs to a host process, and an unrecognized owner says the record states an + // owner this build cannot read; either way, what a stop may do to that pid changes. + status.ownership === 'embedded' + ? 'host-owned' + : status.ownership === 'unknown' + ? 'owner-not-recognized' + : null, + status.version, + status.transport, + ].filter(Boolean); return [ `Daemon running (${details.join(', ')}) for ${status.stateDir}.`, ...(status.plugins.length === 0 diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 12fbdb0894..49d3cbb045 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -514,6 +514,7 @@ Runner and daemon lifecycle (applies to simulators too): Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the 60 s stall window, so a ceiling set below 60 s also ends a silent own launch this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. + A host-owned runtime records ownership and daemon status shows host-owned or owner-not-recognized. Managed clients reuse a matching host-owned runtime, but never replace, signal or retire it after a timeout. A host closes its own runtime directly. daemon stop refuses a live host-owned or unrecognized owner without signaling it; --clean also requires a proven detached owner. Detached stop and cleanup use the lifetime and ownership from one registration read. An unreadable daemon.json fails stop with daemon-record-unreadable. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts new file mode 100644 index 0000000000..82bf05d402 --- /dev/null +++ b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts @@ -0,0 +1,168 @@ +// What a client may do about a runtime a host owns, and what a client pinned to one daemon instance +// will and will not do to reach it. Most of the weight here is in what did NOT happen: nothing was +// signaled, nothing was started, nothing was retired, and no request was dispatched. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { afterEach, test, vi } from 'vitest'; +import { + closeLoopbackServer, + createTestHostEnvironment, + skipWhenLoopbackUnavailable, +} from '@agent-device/test-kit'; +import { DAEMON_LIFECYCLE_REASONS } from '@agent-device/daemon-contracts/daemon-resolution'; +import { AppError } from '@agent-device/kernel/errors'; +import { isProcessAlive } from '@agent-device/host-kit/process'; +import { finishRegisteredDaemonFixtures } from '../../__tests__/test-utils/registered-daemon-fixture.ts'; +import { + launchesFreshDaemonAt, + okResult, + registerDaemon, + startScriptedDaemon, + stateDirAndPaths, +} from './explicit-lifecycle-fixture.ts'; + +vi.mock('@agent-device/host-kit/command', async (importOriginal) => ({ + ...(await importOriginal()), + runCmdDetachedMonitored: vi.fn(), +})); +import { runCmdDetachedMonitored } from '@agent-device/host-kit/command'; +import { createDaemonTransport, type sendToDaemon } from '../daemon-client.ts'; +import { resolveClientSettings, resolveTimeoutRecovery } from '../daemon-client-lifecycle.ts'; +import { readDaemonInfo } from '../daemon-client-metadata.ts'; + +const spawn = vi.mocked(runCmdDetachedMonitored); + +afterEach(async () => { + vi.restoreAllMocks(); + spawn.mockReset(); + await finishRegisteredDaemonFixtures(); +}); + +function request(stateDir: string) { + return { + session: 'default', + command: 'devices', + positionals: [], + flags: { stateDir, daemonTransport: 'http' }, + } as Parameters[0]; +} + +/** + * The signals this code sent at a process. `0` is the liveness probe and is not one; a call with no + * signal argument is SIGTERM, so an omitted signal counts. + */ +function signalsSent(killSpy: { mock: { calls: Array } }) { + return killSpy.mock.calls.filter(([, signal]) => signal !== 0); +} + +async function refusalFrom(promise: Promise): Promise { + const refusal = await promise.then( + () => assert.fail('expected the request to be refused'), + (error: unknown) => error, + ); + assert.ok(refusal instanceof AppError, String(refusal)); + return refusal; +} + +/** + * A stale daemon of an incompatible version: the shape that makes a managed client want this state + * directory. The ownership field is then the only thing that decides whether it may take it. The + * stand-in answers nothing useful on purpose: the embedded refusal happens before the client asks + * this daemon anything. + */ +async function startStaleRegistration(ownership?: 'embedded') { + const { stateDir, paths } = stateDirAndPaths(); + const daemon = await startScriptedDaemon(okResult({})); + const registered = await registerDaemon(paths, { + httpPort: daemon.port, + version: '0.0.0-stale', + capabilities: ['quiesce'], + ...(ownership ? { ownership } : {}), + }); + return { stateDir, paths, daemon, ...registered }; +} + +for (const [ownership, reason, hint] of [ + ['embedded', DAEMON_LIFECYCLE_REASONS.ownerEmbedded, /Close the host/], + ['sideways', 'daemon-owner-unrecognized', /Update agent-device/], +] as const) + test(`a managed client preserves a ${ownership} owner`, async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const { stateDir, paths, daemon, child } = await startStaleRegistration('embedded'); + if (ownership !== 'embedded') { + const record = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')); + fs.writeFileSync(paths.infoPath, JSON.stringify({ ...record, ownership })); + assert.equal(readDaemonInfo(paths.infoPath)?.ownership, 'unknown'); + } + const killSpy = vi.spyOn(process, 'kill'); + try { + const refusal = await refusalFrom( + createDaemonTransport(createTestHostEnvironment())(request(stateDir)), + ); + assert.equal(refusal.details?.reason, reason); + assert.match(String(refusal.details?.hint), hint); + assert.deepEqual(daemon.rpcCommands, [], 'the takeover was never attempted'); + assert.equal(spawn.mock.calls.length, 0); + assert.deepEqual(signalsSent(killSpy), []); + assert.equal(isProcessAlive(child.pid), true); + assert.equal(readDaemonInfo(paths.infoPath)?.pid, child.pid); + } finally { + await closeLoopbackServer(daemon.server); + } + }); + +test('the same record with no host owner is still the takeover this client performs', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + // One field differs from the row above, and it changes everything: a detached contender of the wrong + // version is a daemon this client ends and replaces. That is what makes the embedded refusal a + // boundary rather than a general rule about taking over. + const { stateDir, paths } = stateDirAndPaths(); + const stale = await startScriptedDaemon(okResult({})); + const fresh = await startScriptedDaemon(okResult({ devices: [] })); + const { child } = await registerDaemon(paths, { + httpPort: stale.port, + version: '0.0.0-stale', + capabilities: ['quiesce'], + }); + launchesFreshDaemonAt(spawn, paths, fresh.port); + const killSpy = vi.spyOn(process, 'kill'); + + try { + assert.equal(readDaemonInfo(paths.infoPath)?.ownership, 'detached'); + const response = await createDaemonTransport(createTestHostEnvironment())(request(stateDir)); + + assert.equal(response.ok, true); + assert.equal(spawn.mock.calls.length, 1, 'the detached contender was replaced'); + // The signal the embedded row proves never happens is exactly what this path is allowed to do. + assert.deepEqual( + killSpy.mock.calls.filter(([target, signal]) => target === child.pid && signal === 'SIGTERM'), + [[child.pid, 'SIGTERM']], + ); + } finally { + await closeLoopbackServer(stale.server); + await closeLoopbackServer(fresh.server); + } +}); + +test.each(['unknown', 'embedded', 'detached'] as const)( + 'timeout recovery respects %s ownership', + (ownership) => { + const { stateDir } = stateDirAndPaths(); + const settings = resolveClientSettings( + request(stateDir), + undefined, + createTestHostEnvironment(), + 'managed', + ); + assert.equal( + resolveTimeoutRecovery(settings, { + token: 'token', + pid: 4242, + processStartTime: 'start', + ownership, + }), + ownership === 'detached' ? 'retire-unresponsive' : 'none', + ); + }, +); diff --git a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts index 30375c6bd5..6e307b1a4d 100644 --- a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { closeLoopbackServer, supportsLoopbackBind } from '@agent-device/test-kit'; import { AppError } from '@agent-device/kernel/errors'; -import { DAEMON_QUIESCE_COMMAND } from '@agent-device/daemon-contracts/daemon-resolution'; +import { DAEMON_LIFECYCLE_COMMANDS } from '@agent-device/daemon-contracts/daemon-resolution'; import { DAEMON_STARTUP_EXIT_CODES } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { isProcessAlive } from '@agent-device/host-kit/process'; import { readVersion } from '@agent-device/host-kit/version'; @@ -254,7 +254,7 @@ test('ensureDaemon replaces an incompatible idle daemon through a fenced quiesce }); assert.equal(connection.started, true); - assert.deepEqual(stale.rpcCommands, [DAEMON_QUIESCE_COMMAND]); + assert.deepEqual(stale.rpcCommands, [DAEMON_LIFECYCLE_COMMANDS.quiesce]); assert.equal(stale.rpcRequests[0]?.params.token, 'local-secret'); assert.deepEqual(stale.rpcRequests[0]?.params.input, { pid: info.pid, @@ -485,7 +485,7 @@ test('a live daemon on another transport is quiesced, never killed as unreachabl onStart: (reason) => started.push(reason), }); - assert.deepEqual(live.commands, [DAEMON_QUIESCE_COMMAND]); + assert.deepEqual(live.commands, [DAEMON_LIFECYCLE_COMMANDS.quiesce]); assert.deepEqual(started, ['incompatible']); assert.equal(isProcessAlive(child.pid), false, 'it left because it agreed to'); assert.deepEqual(leaver.clientSignals, [], 'the client signaled nothing'); @@ -543,7 +543,7 @@ test('ensureDaemon judges again after the daemon it addressed turned out to be g assert.equal(typeof actualSend, 'function'); assert.deepEqual( stale.rpcCommands, - [DAEMON_QUIESCE_COMMAND], + [DAEMON_LIFECYCLE_COMMANDS.quiesce], 'the second judgement asked again', ); assert.deepEqual(leaver.clientSignals, [], 'the client signaled nothing'); @@ -648,7 +648,11 @@ test('ensureDaemon quiesces an incompatible daemon that wins a startup race inst const connection = await ensureDaemon({ stateDir, daemonTransport: 'http' }); assert.equal(connection.started, true); - assert.deepEqual(stale.rpcCommands, [DAEMON_QUIESCE_COMMAND], 'the racer was asked to leave'); + assert.deepEqual( + stale.rpcCommands, + [DAEMON_LIFECYCLE_COMMANDS.quiesce], + 'the racer was asked to leave', + ); assert.ok(winner); assert.equal(isProcessAlive(winner.pid), false); assert.equal(spawn.mock.calls.length, 2, 'the second start follows the quiesce'); diff --git a/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts b/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts index d2eebf6867..0a52e50a18 100644 --- a/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts +++ b/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts @@ -125,6 +125,7 @@ type Registration = { socketPort?: number; version?: string; capabilities?: readonly string[]; + ownership?: 'embedded' | 'detached'; }; /** A live child owning the registration, advertising `fields` at the stand-in's port. */ @@ -139,6 +140,7 @@ export async function registerDaemon(paths: DaemonPaths, fields: Registration) { codeOrigin: 'checkout', codeSignature: currentDaemonCodeSignature(), ...(fields.capabilities ? { capabilities: fields.capabilities as ['quiesce'] } : {}), + ...(fields.ownership ? { ownership: fields.ownership } : {}), }, undefined, ); diff --git a/src/daemon-client/daemon-client-ensure.ts b/src/daemon-client/daemon-client-ensure.ts index 3db48b3902..8fa83ec480 100644 --- a/src/daemon-client/daemon-client-ensure.ts +++ b/src/daemon-client/daemon-client-ensure.ts @@ -18,9 +18,9 @@ import { stopDaemonForTakeover, } from './daemon-client-startup.ts'; import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; -import { requestDaemonQuiesce } from './daemon-client-quiesce.ts'; +import { DAEMON_LEAVE_WAIT_MS, requestDaemonLifecycle } from './daemon-client-lifecycle-request.ts'; import { probeDaemonResponsive } from './daemon-client-liveness-probe.ts'; -import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { observedDaemonIdentityOf, type DaemonInfo } from './daemon-client-metadata.ts'; import { DAEMON_CONTENDED_REASON, daemonIdentityOf, @@ -34,10 +34,6 @@ import { // a short wait, for as long as a drain can take. const ENSURE_ATTEMPTS = 40; const JUDGE_AGAIN_DELAY_MS = 250; -// How long a daemon that accepted a quiesce request gets to finish leaving. Its own shutdown is -// bounded in phases (server close, session and lease teardown, provider release), so this is the -// longest a healthy drain can take with margin, not a polling interval. -const DRAIN_DEADLINE_MS = 60_000; const DRAIN_POLL_MS = 250; const UNRESPONSIVE_PROBES = 3; const UNRESPONSIVE_PROBE_GAP_MS = 200; @@ -157,26 +153,27 @@ async function awaitDrainedDaemon( ): Promise<'left' | 'retired'> { const { exited } = await waitForDaemonExit( { pid: info.pid, startTime }, - { timeoutMs: DRAIN_DEADLINE_MS, pollMs: DRAIN_POLL_MS }, + { timeoutMs: DAEMON_LEAVE_WAIT_MS, pollMs: DRAIN_POLL_MS }, ); if (exited) { // The daemon removes its own registration; this only clears what a crashed drain left behind. await recoverAbandonedDaemonRegistration({ paths: settings.paths, - observed: { pid: info.pid, startTime }, + observed: observedDaemonIdentityOf(info), }); return 'left'; } if (await answersLivenessProbe(info)) throw await daemonDrainTimeoutError(info, settings); const retirement = await stopDaemonForTakeover(info, settings.paths, 'force'); if (retirement.status !== 'retained') return 'retired'; - // The identity check could not prove the live process is the daemon it observed (an embedded host - // runs no daemon entry; a start time or command line may also be unreadable), so nothing was - // signaled. - if ( - retirement.termination?.status === 'retained' && - retirement.termination.reason === 'identity-unverified' - ) { + // Nothing was signaled because the process could not be proved to be a daemon of its own: either its + // start time or command line is unreadable, or the record states an owner this build cannot read. + // That is the drain's own outcome: it is still leaving, or already gone, and nobody knows which. A + // refusal that names an owner instead (a host that ended up holding this state directory mid-drain) + // is a replacement that could not be confirmed, and reports as one. + const refusedReason = + retirement.termination?.status === 'retained' ? retirement.termination.reason : undefined; + if (refusedReason === 'identity-unverified' || refusedReason === 'owner-unproven') { throw await daemonDrainTimeoutError(info, settings); } throw daemonRetirementUnconfirmedError(retirement, settings.paths); @@ -191,7 +188,7 @@ async function quiesceOrRetireUnresponsive( settings: DaemonClientSettings, ): Promise<'draining' | 'gone' | 'retired'> { try { - return await requestDaemonQuiesce(info, settings); + return (await requestDaemonLifecycle(info, settings, 'quiesce')).outcome; } catch (error) { if (!(error instanceof AppError) || error.details?.reason !== 'daemon_transport_timeout') throw error; diff --git a/src/daemon-client/daemon-client-lifecycle-request.ts b/src/daemon-client/daemon-client-lifecycle-request.ts new file mode 100644 index 0000000000..0e8771231b --- /dev/null +++ b/src/daemon-client/daemon-client-lifecycle-request.ts @@ -0,0 +1,114 @@ +import { + DAEMON_LIFECYCLE_REASONS, + DAEMON_SHUTTING_DOWN_REASON, + buildDaemonLifecycleRequest, + readDaemonLifecycleCensus, + type DaemonLifecycleMode, + type DaemonPaths, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import type { HostEnvironment } from '@agent-device/kernel/source-value'; +import { createRequestId } from '@agent-device/host-kit/diagnostics'; +import { AppError, throwDaemonError } from '@agent-device/kernel/errors'; +import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { isDaemonTransportUnavailableError, sendRequest } from './daemon-client-transport.ts'; +import { daemonBusyError, daemonNotRetirableError } from './daemon-lifecycle-errors.ts'; + +const LIFECYCLE_REQUEST_TIMEOUT_MS = 10_000; + +/** The bounded close budget after an idle daemon accepts a quiesce. */ +export const DAEMON_LEAVE_WAIT_MS = 60_000; + +/** What a lifecycle request needs besides the daemon it addresses: where and how to reach it. */ +export type DaemonLifecycleTarget = Readonly<{ + paths: DaemonPaths; + host: HostEnvironment; +}>; + +// A refused or reset connection: the daemon left, is leaving, or was never there. Nothing was +// accepted, so judging again is safe. +function isConnectionFailure(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === 'daemon_transport_failure'; +} + +/** Whether the instance accepted the quiesce or is no longer there to answer. */ +export type DaemonLifecycleOutcome = Readonly<{ outcome: 'draining' | 'gone' }>; + +/** Asks the fenced instance to quiesce without timeout recovery or signals. */ +export async function requestDaemonLifecycle( + info: DaemonInfo, + target: DaemonLifecycleTarget, + mode: DaemonLifecycleMode, +): Promise { + const requestId = createRequestId(); + const request = buildDaemonLifecycleRequest({ + mode, + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId, + }); + let response; + try { + response = await sendRequest( + info, + request, + // Whichever transport the daemon serves: a client that cannot reach it on its own preference + // is exactly the one that has to ask it to leave. + 'auto', + target.paths, + LIFECYCLE_REQUEST_TIMEOUT_MS, + { host: target.host, timeoutRecovery: 'none' }, + ); + } catch (error) { + if (isDaemonTransportUnavailableError(error) || isConnectionFailure(error)) + return { outcome: 'gone' }; + // Over HTTP a daemon's refusal arrives as an error carrying its reason, not as the response that + // carries it. The ownership refusal is judged here so an embedded runtime says the same + // thing whichever transport it serves; every other reason keeps the path it shipped with. + const refusal = readOwnerRefusal(error); + const judged = refusal && judgeRefusedAnswer(refusal, info, target); + if (judged) return judged; + throw error; + } + if (response.ok) return { outcome: 'draining' }; + const details = response.error.details; + const judged = judgeRefusedAnswer({ reason: details?.reason, details }, info, target); + if (judged) return judged; + throwDaemonError(response.error); +} + +/** A daemon's refusal, as read from either transport's shape of it. */ +type RefusedDaemonAnswer = Readonly<{ reason: unknown; details: unknown }>; + +/** Ownership refusals have the same meaning over HTTP and sockets. */ +const OWNER_REFUSALS: readonly string[] = [DAEMON_LIFECYCLE_REASONS.ownerEmbedded]; + +function readOwnerRefusal(error: unknown): RefusedDaemonAnswer | undefined { + if (!(error instanceof AppError)) return undefined; + const reason = error.details?.reason; + return typeof reason === 'string' && OWNER_REFUSALS.includes(reason) + ? { reason, details: error.details } + : undefined; +} + +function judgeRefusedAnswer( + refusal: RefusedDaemonAnswer, + info: DaemonInfo, + target: DaemonLifecycleTarget, +): DaemonLifecycleOutcome | undefined { + const stateDir = target.paths.baseDir; + if (refusal.reason === DAEMON_LIFECYCLE_REASONS.busy) { + throw daemonBusyError(info, stateDir, readDaemonLifecycleCensus(refusal.details)); + } + // A detached daemon has an owner that can signal it, and a host owns an embedded runtime: either + // way, this client is not the one that ends it. The daemon's own answer says which owner is on the + // other side of that line, so it is trusted over this process's possibly-stale read of the record. + if (refusal.reason === DAEMON_LIFECYCLE_REASONS.ownerEmbedded) { + throw daemonNotRetirableError({ info, stateDir }); + } + if ( + refusal.reason === DAEMON_LIFECYCLE_REASONS.instanceMismatch || + refusal.reason === DAEMON_SHUTTING_DOWN_REASON + ) { + return { outcome: 'gone' }; + } +} diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index a30fd19bcc..f8df7d5076 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -24,7 +24,7 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import type { findUnrecoveredRepairCommitFailure } from '@agent-device/daemon-contracts/session-repair-tombstone'; import type { DaemonInfo } from './daemon-client-metadata.ts'; import { attachActiveSessionAddressHint } from './daemon-client-address-hints.ts'; -import { isRemoteDaemon } from './daemon-client-metadata.ts'; +import { isRemoteDaemon, observedDaemonIdentityOf } from './daemon-client-metadata.ts'; import { cachedRemoteDaemonHealth } from './daemon-client-transport.ts'; import { assertDaemonPolicyMatches, @@ -60,6 +60,17 @@ LOOPBACK_BLOCK_LIST.addSubnet('127.0.0.0', 8, 'ipv4'); LOOPBACK_BLOCK_LIST.addAddress('::1', 'ipv6'); LOOPBACK_BLOCK_LIST.addSubnet('::ffff:127.0.0.0', 104, 'ipv6'); +export function resolveTimeoutRecovery( + settings: DaemonClientSettings, + info: DaemonInfo, +): 'retire-unresponsive' | 'none' { + if (settings.daemonLifecycle === 'attach') return 'none'; + // Only a record that proves a detached daemon authorizes a kill: an embedded one belongs to a host, + // and one whose owner this build cannot read says nothing about what is safe to signal. + if (info.ownership !== 'detached') return 'none'; + return 'retire-unresponsive'; +} + export function resolveClientSettings( req: Omit, suppliedAuthToken: string | undefined, @@ -283,7 +294,7 @@ export async function cleanupDaemonAfterRequest( const result = await stopAndRetireDaemon({ paths: settings.paths, - observed: { pid: daemon.info.pid, startTime: daemon.info.processStartTime ?? null }, + observed: observedDaemonIdentityOf(daemon.info), mode: 'graceful', ownedStateDir: settings.ownedStateDir, }); diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index 8713947d7a..33c90989f3 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -4,11 +4,14 @@ import { isProcessPid } from '@agent-device/host-kit/process'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import { readDaemonCapabilities, + readDaemonOwnership, readDaemonRecord, readRecordedDaemonTransport, type DaemonCapability, + type DaemonOwnership, type DaemonPaths, type DaemonServerMode, + type ObservedDaemonIdentity, } from '@agent-device/daemon-contracts/daemon-resolution'; export type DaemonInfo = { @@ -25,6 +28,11 @@ export type DaemonInfo = { policyDigest?: string; /** Absent from daemons that predate capabilities, which therefore have none. */ capabilities?: readonly DaemonCapability[]; + /** + * Who owns the process this record names, and so who may end it. A record that predates the field + * is a detached daemon, which is why an absent value means exactly that. + */ + ownership?: DaemonOwnership; /** The provider plugin inventory as published; `readPluginInventory` validates it where it is read. */ plugins?: unknown; baseUrl?: string; @@ -39,12 +47,33 @@ export type DaemonMetadataState = { export function readDaemonInfo(infoPath: string): DaemonInfo | null { const read = readDaemonRecord(infoPath); - if (read.status !== 'present') return null; - const parsed = read.record as Partial; + return read.status === 'present' ? daemonInfoFromRecord(read.record) : null; +} + +function daemonInfoFromRecord(parsed: Partial): DaemonInfo | null { const token = readRequiredDaemonToken(parsed); - if (!token) return null; const ports = readDaemonInfoPorts(parsed); - if (!ports) return null; + return token && ports ? readDaemonInfoFromRecord(parsed, token, ports) : null; +} + +/** + * The observation a retirement judges, from a record this client already read. Ownership travels with + * the lifetime because a retirement signals only a process its record proves is a daemon of its own, and + * a client that read the record once should not read it again to answer that. + */ +export function observedDaemonIdentityOf(info: DaemonInfo): ObservedDaemonIdentity { + return { + pid: info.pid, + startTime: info.processStartTime ?? null, + ownership: readDaemonOwnership(info.ownership), + }; +} + +function readDaemonInfoFromRecord( + parsed: Partial, + token: string, + ports: Pick, +): DaemonInfo { return { token, ...ports, @@ -56,6 +85,7 @@ export function readDaemonInfo(infoPath: string): DaemonInfo | null { processStartTime: readOptionalString(parsed.processStartTime), policyDigest: readOptionalString(parsed.policyDigest), capabilities: readDaemonCapabilities(parsed.capabilities), + ownership: readDaemonOwnership(parsed.ownership), ...(Array.isArray(parsed.plugins) ? { plugins: parsed.plugins } : {}), }; } diff --git a/src/daemon-client/daemon-client-quiesce.ts b/src/daemon-client/daemon-client-quiesce.ts deleted file mode 100644 index 6a65992624..0000000000 --- a/src/daemon-client/daemon-client-quiesce.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - DAEMON_QUIESCE_REASONS, - DAEMON_SHUTTING_DOWN_REASON, - buildDaemonQuiesceRequest, - readDaemonQuiesceCensus, -} from '@agent-device/daemon-contracts/daemon-resolution'; -import { createRequestId } from '@agent-device/host-kit/diagnostics'; -import { AppError, throwDaemonError } from '@agent-device/kernel/errors'; -import type { DaemonInfo } from './daemon-client-metadata.ts'; -import type { DaemonClientSettings } from './daemon-client-lifecycle.ts'; -import { isDaemonTransportUnavailableError, sendRequest } from './daemon-client-transport.ts'; -import { daemonBusyError } from './daemon-lifecycle-errors.ts'; - -const QUIESCE_TIMEOUT_MS = 10_000; - -// A refused or reset connection: the daemon left, is leaving, or was never there. Nothing was -// accepted, so judging again is safe. -function isConnectionFailure(error: unknown): boolean { - return error instanceof AppError && error.details?.reason === 'daemon_transport_failure'; -} - -/** - * `draining`: the daemon accepted and is leaving. `gone`: the daemon the request was addressed to - * is no longer there to answer, because it left, is leaving already, or was replaced. - */ -export type QuiesceOutcome = 'draining' | 'gone'; - -/** - * Asks the daemon to leave, fenced to the instance `info` names. A daemon that holds work refuses - * with `daemon-busy`, and that refusal is the answer: nothing here signals it. A timeout is not - * recovered here either; the caller decides what an unresponsive daemon deserves. - */ -export async function requestDaemonQuiesce( - info: DaemonInfo, - settings: DaemonClientSettings, -): Promise { - const request = buildDaemonQuiesceRequest({ - token: info.token, - fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, - requestId: createRequestId(), - }); - let response; - try { - response = await sendRequest( - info, - request, - // Whichever transport the daemon serves: a client that cannot reach it on its own preference - // is exactly the one that has to ask it to leave. - 'auto', - settings.paths, - QUIESCE_TIMEOUT_MS, - { host: settings.host, timeoutRecovery: 'none' }, - ); - } catch (error) { - if (isDaemonTransportUnavailableError(error) || isConnectionFailure(error)) return 'gone'; - throw error; - } - if (response.ok) return 'draining'; - const reason = response.error.details?.reason; - if (reason === DAEMON_QUIESCE_REASONS.busy) { - throw daemonBusyError( - info, - settings.paths.baseDir, - readDaemonQuiesceCensus(response.error.details), - ); - } - if ( - reason === DAEMON_QUIESCE_REASONS.instanceMismatch || - reason === DAEMON_SHUTTING_DOWN_REASON - ) { - return 'gone'; - } - return throwDaemonError(response.error); -} diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index b02220d85c..ebd6f7e6d4 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -32,6 +32,7 @@ import { } from './daemon-launch-spec.ts'; import { getDaemonMetadataState, + observedDaemonIdentityOf, readDaemonInfo, resolveDaemonStartupHint, type DaemonInfo, @@ -39,6 +40,8 @@ import { import { canConnect, isDaemonTransportUnavailableError } from './daemon-client-transport.ts'; import { daemonContendedError, + daemonNotRetirableError, + daemonRecordOwnerUnrecognizedError, daemonRetirementUnconfirmedError, daemonStopCommand, } from './daemon-lifecycle-errors.ts'; @@ -149,6 +152,22 @@ export async function readReusableLocalDaemon( } if (!settings.retireIncompatibleContender) throw daemonContendedError(); + // A host owns an embedded runtime, so a managed client that wants this state directory asks the host + // for it instead of retiring the runtime that holds it. + if (existing.ownership === 'embedded') { + throw daemonNotRetirableError({ + info: existing, + stateDir: settings.paths.baseDir, + }); + } + // A record stating an owner this build cannot read is not a licence to replace what it names: the + // whole point of reading the owner is to know whether a pid may be signaled, and this says nothing. + if (existing.ownership === 'unknown') { + throw daemonRecordOwnerUnrecognizedError({ + info: existing, + stateDir: settings.paths.baseDir, + }); + } if (remainingStartupBudget(deadline) < MINIMUM_DAEMON_TAKEOVER_BUDGET_MS) return null; emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); await retireDaemonForTakeover(existing, settings.paths); @@ -196,7 +215,7 @@ export async function stopDaemonForTakeover( ): Promise { return await stopAndRetireDaemon({ paths: paths, - observed: { pid: existing.pid, startTime: existing.processStartTime ?? null }, + observed: observedDaemonIdentityOf(existing), mode, }); } @@ -434,7 +453,7 @@ async function retireStartupAttempt( ): Promise<{ cleanup: DaemonRetirementResult; joined: boolean }> { const cleanup = await stopAndRetireDaemon({ paths: settings.paths, - observed: launch.readIdentity(), + observed: { ...launch.readIdentity(), ownership: 'detached' as const }, mode: 'graceful', ownedStateDir: settings.ownedStateDir, termTimeoutMs: Math.min(3_000, remainingStartupBudget(deadline)), diff --git a/src/daemon-client/daemon-client-stop.ts b/src/daemon-client/daemon-client-stop.ts new file mode 100644 index 0000000000..a1f69775e5 --- /dev/null +++ b/src/daemon-client/daemon-client-stop.ts @@ -0,0 +1,58 @@ +import { readDaemonShutdownReport } from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import type { DaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { + stopDaemon, + mergeDaemonShutdownReport, + type DaemonStopResult, +} from '@agent-device/daemon-contracts/daemon-stop'; +import type { DaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import type { HostEnvironment } from '@agent-device/kernel/source-value'; +import { AppError } from '@agent-device/kernel/errors'; + +export async function stopRecordedDaemon( + record: DaemonStopIdentity, + target: { paths: DaemonPaths; host: HostEnvironment }, +): Promise { + if (record.ownership === 'unreadable') { + throw new AppError( + 'COMMAND_FAILED', + `The daemon record on ${target.paths.baseDir} cannot be read, so this stop cannot tell whether a daemon runs.`, + { + reason: 'daemon-record-unreadable', + stateDir: target.paths.baseDir, + hint: 'Nothing was signaled. Inspect or remove that record, then retry.', + }, + ); + } + const observed = + record.identity && + (record.ownership === 'embedded' || + record.ownership === 'unknown' || + record.ownership === 'detached') + ? { ...record.identity, ownership: record.ownership } + : null; + let stopped: DaemonStopResult; + try { + stopped = await stopDaemon({ paths: target.paths, observed }); + } catch (error) { + if ( + error instanceof AppError && + (error.details?.terminationReason === 'embedded-owner' || + error.details?.terminationReason === 'owner-unproven') + ) { + throw new AppError(error.code, error.message, { + ...error.details, + reason: + error.details.terminationReason === 'embedded-owner' + ? 'daemon-owner-embedded' + : 'daemon-owner-unrecognized', + hint: 'Nothing was signaled. Close the host that started this runtime, or repair its record after the process has exited.', + }); + } + throw error; + } + return mergeDaemonShutdownReport( + stopped, + stopped.mode === 'graceful' ? readDaemonShutdownReport(target.paths.baseDir) : null, + ); +} diff --git a/src/daemon-client/daemon-client-timeout.ts b/src/daemon-client/daemon-client-timeout.ts index 4b33a04b40..550e7d3887 100644 --- a/src/daemon-client/daemon-client-timeout.ts +++ b/src/daemon-client/daemon-client-timeout.ts @@ -5,7 +5,7 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { resolveCommandTimeoutPolicy } from '@agent-device/command-registry/registry'; import type { DaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import type { PlatformSelector } from '@agent-device/kernel/device'; -import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { observedDaemonIdentityOf, type DaemonInfo } from './daemon-client-metadata.ts'; // Recovery for a timed-out request is scoped to that request or to nothing. The daemon owns // request-scoped recovery (#3177): destroying the timed-out connection cancels exactly that @@ -120,7 +120,7 @@ async function forceRetireDaemon( await import('@agent-device/daemon-contracts/daemon-registration-owner'); return await stopAndRetireDaemon({ paths: statePaths, - observed: { pid: info.pid, startTime: info.processStartTime ?? null }, + observed: observedDaemonIdentityOf(info), mode: 'force', }); } diff --git a/src/daemon-client/daemon-client.ts b/src/daemon-client/daemon-client.ts index 22d4a5b7a7..2b09e75e64 100644 --- a/src/daemon-client/daemon-client.ts +++ b/src/daemon-client/daemon-client.ts @@ -64,7 +64,7 @@ async function sendToDaemonOnHost( options: DaemonTransportOptions, host: HostEnvironment, ): Promise { - const { resolveClientSettings, attachSessionAddressHints } = + const { resolveClientSettings, resolveTimeoutRecovery, attachSessionAddressHints } = await import('./daemon-client-lifecycle.ts'); const { connectToDaemon } = await import('./daemon-client-attach.ts'); const requestId = req.meta?.requestId ?? createRequestId(); @@ -149,8 +149,7 @@ async function sendToDaemonOnHost( host, onProgress: options.onProgress, signal: options.signal, - timeoutRecovery: - settings.daemonLifecycle === 'attach' ? 'none' : 'retire-unresponsive', + timeoutRecovery: resolveTimeoutRecovery(settings, info), }, ), { requestId, command: req.command }, diff --git a/src/daemon-client/daemon-lifecycle-errors.ts b/src/daemon-client/daemon-lifecycle-errors.ts index 3f1e2de7dc..54bde7aa6f 100644 --- a/src/daemon-client/daemon-lifecycle-errors.ts +++ b/src/daemon-client/daemon-lifecycle-errors.ts @@ -1,8 +1,9 @@ import type { DaemonIdentity } from '@agent-device/contracts/client'; import { - DAEMON_QUIESCE_REASONS, + DAEMON_LIFECYCLE_REASONS, + DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON, + type DaemonLifecycleCensus, type DaemonPaths, - type DaemonQuiesceCensus, } from '@agent-device/daemon-contracts/daemon-resolution'; import type { DaemonRetirementResult } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { AppError } from '@agent-device/kernel/errors'; @@ -16,12 +17,14 @@ import { } from './daemon-client-metadata.ts'; import { resolveLocalDaemonCodeIdentity, type DaemonTakeoverCause } from './daemon-launch-spec.ts'; -// The four ways an explicit lifecycle stops short of a usable daemon. Callers branch on -// `details.reason`, so each reason is one constant here and the messages are free to change. -const DAEMON_LIFECYCLE_REASONS = { +// The ways an explicit lifecycle stops short of a usable daemon, whether this client judged it or the +// daemon answered it: `busy` is the daemon's own refusal, carried through so one constant names it on +// both sides. Callers branch on `details.reason`, so each reason is one constant here and the messages +// are free to change. +const DAEMON_UNAVAILABLE_REASONS = { notRunning: 'daemon-not-running', incompatible: 'daemon-incompatible', - busy: DAEMON_QUIESCE_REASONS.busy, + busy: DAEMON_LIFECYCLE_REASONS.busy, drainTimeout: 'daemon-drain-timeout', } as const; @@ -64,7 +67,7 @@ export function daemonContendedError(): AppError { export function daemonNotRunningError(stateDir: string, registered?: DaemonInfo): AppError { return new AppError('COMMAND_FAILED', 'No daemon is running for this state directory.', { - reason: DAEMON_LIFECYCLE_REASONS.notRunning, + reason: DAEMON_UNAVAILABLE_REASONS.notRunning, stateDir, ...(registered ? { daemon: daemonIdentityOf(registered) } : {}), hint: `Start one with a local agent-device command that uses the daemon (agent-device devices --state-dir ${shellQuoteIfNeeded(stateDir)}, for example), or with ensureDaemon({ stateDir: ${JSON.stringify(stateDir)} }) from the SDK, before attaching.${registered ? ` The registered daemon (pid ${registered.pid}) does not answer; ${daemonStopCommand(stateDir)} clears it.` : ''}`, @@ -83,7 +86,9 @@ export function daemonRetirementUnconfirmedError( } export function isDaemonNotRunningError(error: unknown): boolean { - return error instanceof AppError && error.details?.reason === DAEMON_LIFECYCLE_REASONS.notRunning; + return ( + error instanceof AppError && error.details?.reason === DAEMON_UNAVAILABLE_REASONS.notRunning + ); } export async function daemonIncompatibleError(params: { @@ -93,7 +98,7 @@ export async function daemonIncompatibleError(params: { }): Promise { const { info, mismatch, stateDir } = params; return new AppError('COMMAND_FAILED', 'The running daemon cannot serve this client.', { - reason: DAEMON_LIFECYCLE_REASONS.incompatible, + reason: DAEMON_UNAVAILABLE_REASONS.incompatible, mismatch, stateDir, daemon: daemonIdentityOf(info), @@ -111,7 +116,7 @@ export async function daemonDrainTimeoutError( 'COMMAND_FAILED', 'The daemon accepted the request to leave but is still running.', { - reason: DAEMON_LIFECYCLE_REASONS.drainTimeout, + reason: DAEMON_UNAVAILABLE_REASONS.drainTimeout, stateDir, daemon: daemonIdentityOf(info), hint: `It was not signaled. Retry once it has exited, or stop it deliberately with ${daemonStopCommand(stateDir)}.`, @@ -122,10 +127,10 @@ export async function daemonDrainTimeoutError( export function daemonBusyError( info: DaemonInfo, stateDir: string, - census: DaemonQuiesceCensus | undefined, + census: DaemonLifecycleCensus | undefined, ): AppError { return new AppError('COMMAND_FAILED', 'The running daemon is busy and was not replaced.', { - reason: DAEMON_LIFECYCLE_REASONS.busy, + reason: DAEMON_UNAVAILABLE_REASONS.busy, stateDir, daemon: daemonIdentityOf(info), ...census, @@ -134,3 +139,35 @@ export function daemonBusyError( : `Wait for its requests, sessions and leases to end and retry, or stop it deliberately with ${daemonStopCommand(stateDir)}.`, }); } + +/** Refuses managed replacement when the record cannot prove a detached owner. */ +export function daemonRecordOwnerUnrecognizedError(params: { + info: DaemonInfo; + stateDir: string; +}): AppError { + const { info, stateDir } = params; + return new AppError( + 'COMMAND_FAILED', + 'This daemon record states an owner this version cannot read, so its daemon was not replaced.', + { + reason: DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON, + stateDir, + daemon: daemonIdentityOf(info), + hint: `Update agent-device if the record comes from a newer build. Otherwise check what owns pid ${info.pid}, then repair the record: stop that process by its own means and remove ${shellQuoteIfNeeded(stateDir)}/daemon.json so a daemon can register again.`, + }, + ); +} + +/** Refuses replacement of a runtime whose host owns its lifetime. */ +export function daemonNotRetirableError(params: { info: DaemonInfo; stateDir: string }): AppError { + return new AppError( + 'COMMAND_FAILED', + 'A host process owns this daemon, so it was not replaced.', + { + reason: DAEMON_LIFECYCLE_REASONS.ownerEmbedded, + stateDir: params.stateDir, + daemon: daemonIdentityOf(params.info), + hint: 'Close the host that started this runtime.', + }, + ); +} diff --git a/test/integration/smoke-daemon-clean.test.ts b/test/integration/smoke-daemon-clean.test.ts index 017c532faf..c2e0199020 100644 --- a/test/integration/smoke-daemon-clean.test.ts +++ b/test/integration/smoke-daemon-clean.test.ts @@ -64,6 +64,34 @@ test('clean daemon script stops a live daemon before removing metadata', async ( info = readDaemonInfo(stateDir); assert.equal(isProcessAlive(info.pid), true); + const infoPath = path.join(stateDir, 'daemon.json'); + const original = fs.readFileSync(infoPath, 'utf8'); + for (const ownership of ['embedded', 'unrecognized']) { + const record = JSON.stringify({ ...JSON.parse(original), ownership }); + fs.writeFileSync(infoPath, record); + const refused = runCmdSync( + process.execPath, + ['--experimental-strip-types', 'scripts/clean-daemon.ts'], + { + env: { ...process.env, AGENT_DEVICE_STATE_DIR: stateDir }, + timeoutMs: 5_000, + allowFailure: true, + }, + ); + assert.notEqual(refused.exitCode, 0); + assert.match(refused.stderr, ownership === 'embedded' ? /embedded-owner/ : /owner-unproven/); + assert.equal( + isProcessAlive(info.pid), + true, + 'the same otherwise signalable daemon stays alive', + ); + assert.equal(fs.readFileSync(infoPath, 'utf8'), record); + assert.equal(fs.existsSync(path.join(stateDir, 'daemon.lock')), true); + } + const legacy = JSON.parse(original); + delete legacy.ownership; + fs.writeFileSync(infoPath, JSON.stringify(legacy)); + const cleanup = runCmdSync( process.execPath, ['--experimental-strip-types', 'scripts/clean-daemon.ts'], @@ -96,7 +124,13 @@ test('forced retirement waits for a real daemon and reclaims its abandoned regis assert.equal(fs.existsSync(paths.lockPath), true); const result = await stopAndRetireDaemon({ paths, - observed: { pid: info.pid, startTime: info.processStartTime ?? null }, + // The record is the one this row's CLI-started daemon published of itself, which is a detached + // daemon: what the row retires is that lifetime, under that owner. + observed: { + pid: info.pid, + startTime: info.processStartTime ?? null, + ownership: 'detached', + }, mode: 'force', }); assert.equal(result.status, 'retired', JSON.stringify(result)); diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 7fa0b9daf3..50b3a54759 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -90,6 +90,8 @@ if ('cloudArtifacts' in result) { A client talks to one local daemon per state directory. `daemonLifecycle` says who owns that daemon. +A host-owned runtime records its ownership. A managed client reuses it while it matches, but never replaces, signals or retires it after a timeout. The host closes its own runtime directly. `daemon stop` refuses a live host-owned or unrecognized owner without signaling it; `--clean` also requires a proven detached owner. An unreadable `daemon.json` fails stop with `daemon-record-unreadable`. + | Mode | What the client does with the daemon | | --- | --- | | `'managed'` (default) | Starts a missing daemon, replaces one that does not match the client, and retires an unresponsive one after a timeout. | diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 223fdc646a..59f46b983e 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -85,7 +85,7 @@ Device claims and the daemon: - `device status` lists host-local device claims without starting or contacting a daemon; `--stale` shows the stale claims it hides by default (owner exited, PID reused, state directory deleted, or daemon replaced), some of whose owners may still be running. Scope it with `--platform` plus `--udid` (Apple) or `--serial` (Android). A claim held by another live or uncertain owner blocks `open`, unless the device rebooted after the claim was taken: then `open` settles that owner's durable resources, takes the claim, and reports the release in `warnings`. - `device release --stale` settles the durable resources of an owner that provably can't release its claim (its process exited or a newer daemon replaced it), then clears the claim. Live, uncertain, PID-reused, corrupt, and deleted-state-directory claims are refused with the reason. See [Sessions](/docs/sessions#release-a-device-another-session-or-workspace-holds). - A command that needs the shared local daemon starts it for its state directory when none runs; `device status` and `device release --stale` never contact one. When the running daemon comes from another version or build, the command replaces it only if it reports no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise the command fails with `daemon-busy` and leaves the daemon alone. A daemon from a release that predates this check is never replaced, and fails with `daemon-incompatible`. End a busy or older daemon deliberately with `daemon stop`. Node.js clients choose this per client; see [Daemon lifecycle](/docs/client-api#daemon-lifecycle). -- `daemon stop --state-dir ` stops a local daemon after verifying its identity. `--clean` also removes the Apple runner processes and leases that daemon retained. +- `daemon stop --state-dir ` stops a detached local daemon after verifying its identity. Stop and `--clean` use the lifetime and ownership from one registration read. A live host-owned or unrecognized owner is refused without signaling; its host closes the runtime directly. An unreadable `daemon.json` fails stop with `daemon-record-unreadable`. `daemon status` shows `host-owned` or `owner-not-recognized`. `--clean` also removes the Apple runner processes and leases that daemon retained. - `daemon status [--state-dir ] [--json]` reads the daemon's record without contacting it. For a running daemon it lists every provider it registered, built in or installed, and the reason any plugin failed to register. See [provider plugins](/docs/plugins). Open apps and links: From 62148c8fd72ed8ea9ec09467404c11f126f0d212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 08:40:35 +0200 Subject: [PATCH 2/4] chore(gates): typecheck the daemon cleanup script --- tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig.json b/tsconfig.json index 52fd1e06cd..f0883f27f3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,6 +26,7 @@ "src", "test", "scripts/help-conformance-command-validator.ts", + "scripts/clean-daemon.ts", "scripts/coverage-crap/**/*.ts", "scripts/coverage-changed/**/*.ts", "scripts/vitest-coverage-run-stamp-reporter.ts", From f5b44dfa87144ee232aebc0c2cdf65c6c4ad9b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 15:05:18 +0200 Subject: [PATCH 3/4] fix(daemon): judge takeover ownership once inside the retirement readReusableLocalDaemon rejected an embedded or unknown owner before the central retirement could look at it, so a record whose owner had already exited wedged every managed command instead of recovering. Delete the two client guards and the resolveTimeoutRecovery ownership guard: ownership is judged inside stopAndRetireDaemon on every takeover route (readReusableLocalDaemon, replaceUnreachable, quiesceOrRetireUnresponsive, awaitDrainedDaemon and forceRetireDaemon alike). Map a retained owner reason to the existing typed errors once, in daemonRetirementError for the stop route and daemonTakeoverRetirementError for a takeover, and prove the managed route recovers a crashed embedded or unrecognized owner without signaling it. https://github.com/callstack/agent-device/pull/3806#issuecomment-6262194144 --- .../src/daemon-registration.ts | 99 +++++++++++-------- packages/daemon-contracts/src/daemon-stop.ts | 65 +++++++++--- scripts/clean-daemon.ts | 16 ++- ...daemon-registration-embedded-owner.test.ts | 12 +-- src/cli/commands/__tests__/daemon.test.ts | 47 +++++---- src/cli/commands/daemon.ts | 2 +- src/commands/schema/cli-help.ts | 2 +- .../daemon-client-embedded-lifecycle.test.ts | 94 ++++++++++++++---- src/daemon-client/daemon-client-lifecycle.ts | 9 +- src/daemon-client/daemon-client-startup.ts | 23 +---- src/daemon-client/daemon-client-stop.ts | 51 ++++------ src/daemon-client/daemon-client.ts | 2 +- src/daemon-client/daemon-lifecycle-errors.ts | 20 +++- website/docs/docs/client-api.md | 2 +- 14 files changed, 265 insertions(+), 179 deletions(-) diff --git a/packages/daemon-contracts/src/daemon-registration.ts b/packages/daemon-contracts/src/daemon-registration.ts index e1f7ca96f0..12dfbc8402 100644 --- a/packages/daemon-contracts/src/daemon-registration.ts +++ b/packages/daemon-contracts/src/daemon-registration.ts @@ -15,6 +15,7 @@ import { type DaemonOwnership, type DaemonPaths, type DaemonServerMode, + type ObservedDaemonIdentity, } from './daemon-resolution.ts'; /** @@ -24,8 +25,8 @@ import { * however alive it is. */ export function readRegisteredDaemonIdentity(infoPath: string): OwnerIdentity | null { - const record = readRegistration(infoPath); - return record.status === 'registered' ? record.identity : null; + const reading = readDaemonRegistration(infoPath); + return reading.status === 'present' ? reading.identity : null; } export function processLockHoldsDaemonIdentity( @@ -46,26 +47,29 @@ type ParsedRegistration = { pid: number | null; startTime: string | null }; type DaemonRecord = Readonly>; /** - * Whether `daemon.json` registers a daemon: a projection of the one record reader that adds the - * identity, so a record that names no owner is `ownerless` however it came to. + * The one read of `daemon.json` behind every projection of that record in this module. `identity` is + * null for a record that names no process, and `ownership` is what the record states, read the way a + * reader must read it before anything is signaled. */ -function readRegistration( - infoPath: string, -): - | Readonly<{ status: 'registered'; identity: OwnerIdentity; record: DaemonRecord }> - | Readonly<{ status: 'absent' | 'unreadable' | 'ownerless' }> { +type DaemonRegistrationReading = + | Readonly<{ + status: 'present'; + record: DaemonRecord; + identity: OwnerIdentity | null; + ownership: DaemonOwnership; + }> + | Readonly<{ status: 'absent' | 'unreadable' | 'invalid' }>; + +function readDaemonRegistration(infoPath: string): DaemonRegistrationReading { const read = readDaemonRecord(infoPath); - if (read.status !== 'present') { - return { status: read.status === 'invalid' ? 'ownerless' : read.status }; - } + if (read.status !== 'present') return { status: read.status }; const parsed = parseRegistration(read.record); - return parsed.pid === null - ? { status: 'ownerless' } - : { - status: 'registered', - identity: { pid: parsed.pid, startTime: parsed.startTime }, - record: read.record, - }; + return { + status: 'present', + record: read.record, + identity: parsed.pid === null ? null : { pid: parsed.pid, startTime: parsed.startTime }, + ownership: readDaemonOwnership(read.record.ownership), + }; } function parseRegistration(parsed: { @@ -102,15 +106,18 @@ export function readRegisteredDaemonOwnership( infoPath: string, owner: OwnerIdentity | null, ): RegisteredDaemonOwnership { - const record = readRegistration(infoPath); - if (record.status !== 'registered') return { state: record.status }; + const reading = readDaemonRegistration(infoPath); + if (reading.status !== 'present') + return { state: reading.status === 'invalid' ? 'ownerless' : reading.status }; + // A record that names no process describes nothing a caller can own, however it came to that. + if (!reading.identity) return { state: 'ownerless' }; if (!owner) return { state: 'unproven' }; - if (ownerIdentityDiffers(record.identity, owner)) { - return { state: 'replaced', identity: record.identity }; + if (ownerIdentityDiffers(reading.identity, owner)) { + return { state: 'replaced', identity: reading.identity }; } // Two missing start times agree, and agreeing on nothing is not ownership: this is the pid-only // fallback the fence exists to refuse, so it falls to `unproven` rather than to `match`. - if (ownerIdentityMatches(record.identity, owner) && record.identity.startTime !== null) { + if (ownerIdentityMatches(reading.identity, owner) && reading.identity.startTime !== null) { return { state: 'match' }; } return { state: 'unproven' }; @@ -152,16 +159,15 @@ export function isUnusableRecord(status: Pick): boolean { */ export function readDaemonStatus(paths: DaemonPaths): DaemonStatus { const base = { stateDir: paths.baseDir } as const; - const read = readDaemonRecord(paths.infoPath); - if (read.status !== 'present') { - return read.status === 'absent' + const reading = readDaemonRegistration(paths.infoPath); + if (reading.status !== 'present') { + return reading.status === 'absent' ? { state: 'stopped', ...base } - : { state: read.status, ...base }; + : { state: reading.status, ...base }; } - const { record } = read; - const parsed = parseRegistration(record); - if (parsed.pid === null) return { state: 'stopped', ...base }; - const identity = { pid: parsed.pid, startTime: parsed.startTime }; + const { record } = reading; + if (reading.identity === null) return { state: 'stopped', ...base }; + const identity = reading.identity; if (identity.startTime === null || !isProcessAlive(identity.pid)) { return { state: 'stopped', ...base }; } @@ -178,7 +184,7 @@ export function readDaemonStatus(paths: DaemonPaths): DaemonStatus { state: 'running', stateDir: paths.baseDir, pid: identity.pid, - ownership: readDaemonOwnership(record.ownership), + ownership: reading.ownership, ...(typeof record.version === 'string' ? { version: record.version } : {}), ...(transport ? { transport } : {}), plugins: record.plugins, @@ -211,16 +217,23 @@ export type DaemonStopIdentity = Readonly<{ * record written before the ownership field named. */ export function readDaemonStopIdentity(infoPath: string): DaemonStopIdentity { - const read = readDaemonRecord(infoPath); - if (read.status !== 'present') { - return { - ownership: read.status === 'absent' ? 'absent' : 'unreadable', - identity: null, - }; - } - const parsed = parseRegistration(read.record); - const identity = parsed.pid === null ? null : { pid: parsed.pid, startTime: parsed.startTime }; - return { ownership: readDaemonOwnership(read.record.ownership), identity }; + const reading = readDaemonRegistration(infoPath); + if (reading.status !== 'present') + return { ownership: reading.status === 'absent' ? 'absent' : 'unreadable', identity: null }; + return { ownership: reading.ownership, identity: reading.identity }; +} + +/** + * The observation a retirement judges, from a stop's read of the record. `absent` names no process at + * all, and `unreadable` describes one this read cannot use, so neither travels as an owner; the stop + * route refuses the unreadable record before it gets here. + */ +export function observedDaemonIdentityOfRegistration( + record: DaemonStopIdentity, +): ObservedDaemonIdentity | null { + if (!record.identity || record.ownership === 'absent' || record.ownership === 'unreadable') + return null; + return { ...record.identity, ownership: record.ownership }; } /** diff --git a/packages/daemon-contracts/src/daemon-stop.ts b/packages/daemon-contracts/src/daemon-stop.ts index 4302065416..3158c1ef14 100644 --- a/packages/daemon-contracts/src/daemon-stop.ts +++ b/packages/daemon-contracts/src/daemon-stop.ts @@ -1,7 +1,12 @@ import { AppError } from '@agent-device/kernel/errors'; import type { DaemonRetirementResult } from './daemon-registration-owner.ts'; -import type { DaemonPaths, ObservedDaemonIdentity } from './daemon-resolution.ts'; +import { + DAEMON_LIFECYCLE_REASONS, + DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON, + type DaemonPaths, + type ObservedDaemonIdentity, +} from './daemon-resolution.ts'; import type { OwnerIdentity } from '@agent-device/host-kit/process'; import type { DaemonShutdownReport, @@ -43,8 +48,9 @@ export type DaemonStopResult = { * Retires the daemon whose lifetime the caller read, and shapes that retirement as a stop result. * * The caller hands over the lifetime and the owner its own read proved, so routing and retirement judge - * one record. Only a detached daemon reaches here: a host owns an embedded runtime and ends it with its - * own close, and a record this build cannot place proves nothing about what is safe to signal. + * one record. Ownership is judged here and nowhere else: an embedded runtime belongs to its host, and a + * record this build cannot place proves nothing about what is safe to signal, so both come back as the + * refusal that names their owner, with nothing signaled. */ export async function stopDaemon(params: { paths: DaemonPaths; @@ -95,6 +101,25 @@ export async function stopDaemon(params: { }; } +/** + * What a retirement that refused because of the record's owner answers with. This is the one place + * those two retained reasons become a stop's own reason: a caller that may not signal this process + * hears the owner's rule, not a retirement that could not be confirmed. + */ +function ownerRefusal( + reason: 'embedded-owner' | 'owner-unproven', +): Readonly<{ reason: string; hint: string }> { + return reason === 'embedded-owner' + ? { + reason: DAEMON_LIFECYCLE_REASONS.ownerEmbedded, + hint: 'Nothing was signaled. Close the host that started this runtime, or repair its record after the process has exited.', + } + : { + reason: DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON, + hint: 'Nothing was signaled. Check what owns that process, then repair the record: stop it by its own means and remove its daemon.json so a daemon can register again.', + }; +} + function daemonRetirementError( info: OwnerIdentity, retirement: Exclude, @@ -103,17 +128,29 @@ function daemonRetirementError( const failure = termination?.status === 'retained' ? termination : undefined; const error = retirement.status === 'retained' ? retirement.error : undefined; const { hint, diagnosticId, logPath } = error ?? {}; - return new AppError('COMMAND_FAILED', 'Daemon retirement could not be confirmed.', { - pid: info.pid, - processStartTime: info.startTime, - reason: failure ? 'daemon_exit_unconfirmed' : 'daemon_retirement_unconfirmed', - terminationReason: failure?.reason, - signal: failure?.signal, - retirement, - hint, - diagnosticId, - logPath, - }); + const refusal = + retirement.status === 'retained' && + (retirement.reason === 'embedded-owner' || retirement.reason === 'owner-unproven') + ? ownerRefusal(retirement.reason) + : undefined; + return new AppError( + 'COMMAND_FAILED', + refusal + ? 'The daemon was not stopped, because its owner ends it.' + : 'Daemon retirement could not be confirmed.', + { + pid: info.pid, + processStartTime: info.startTime, + reason: + refusal?.reason ?? (failure ? 'daemon_exit_unconfirmed' : 'daemon_retirement_unconfirmed'), + terminationReason: failure?.reason, + signal: failure?.signal, + retirement, + hint: hint ?? refusal?.hint, + diagnosticId, + logPath, + }, + ); } function daemonNotRunningStopResult(): DaemonStopResult { diff --git a/scripts/clean-daemon.ts b/scripts/clean-daemon.ts index 0cc0d986b1..2b3bad630c 100644 --- a/scripts/clean-daemon.ts +++ b/scripts/clean-daemon.ts @@ -5,7 +5,10 @@ import { AppError } from '@agent-device/kernel/errors'; import { withCommandHost } from '@agent-device/host-kit/command'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { + observedDaemonIdentityOfRegistration, + readDaemonStopIdentity, +} from '@agent-device/daemon-contracts/daemon-registration'; import { stopAndRetireDaemon, recoverAbandonedDaemonRegistration, @@ -22,7 +25,7 @@ const paths = resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR, { }); const retirement = await stopAndRetireDaemon({ paths, - observed: readCleanupIdentity(paths.infoPath), + observed: observedDaemonIdentityOfRegistration(readDaemonStopIdentity(paths.infoPath)), mode: 'graceful', termTimeoutMs: DAEMON_TERM_TIMEOUT_MS, killTimeoutMs: DAEMON_KILL_TIMEOUT_MS, @@ -65,7 +68,7 @@ async function pruneStaleDevStateDirs(): Promise { const paths = resolveDaemonPaths(dirPath, { cwd: process.cwd() }); const result = await recoverAbandonedDaemonRegistration({ paths, - observed: readCleanupIdentity(paths.infoPath), + observed: observedDaemonIdentityOfRegistration(readDaemonStopIdentity(paths.infoPath)), }); if (result.status === 'retired') process.stdout.write( @@ -107,10 +110,3 @@ function statMtimeMs(filePath: string): number { return 0; } } - -function readCleanupIdentity(infoPath: string) { - const { identity, ownership } = readDaemonStopIdentity(infoPath); - return identity && ownership !== 'absent' && ownership !== 'unreadable' - ? { ...identity, ownership } - : null; -} diff --git a/src/__tests__/daemon-registration-embedded-owner.test.ts b/src/__tests__/daemon-registration-embedded-owner.test.ts index 76b39764b3..707fa332d1 100644 --- a/src/__tests__/daemon-registration-embedded-owner.test.ts +++ b/src/__tests__/daemon-registration-embedded-owner.test.ts @@ -7,7 +7,7 @@ import { stopAndRetireDaemon } from '@agent-device/daemon-contracts/daemon-regis import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { stopRecordedDaemon } from '../daemon-client/daemon-client-stop.ts'; import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; -import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; // What the registration owner may do to a record a host process owns: prove the lifetime is gone // before removing anything, and never signal the process the record names. @@ -100,10 +100,7 @@ test('retirement of an embedded record proves the lifetime is gone and signals n } await assert.rejects( - stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), { - paths, - host: createTestHostEnvironment(), - }), + stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), paths), (error: unknown) => error instanceof AppError && error.details?.reason === 'daemon-owner-embedded' && @@ -185,10 +182,7 @@ test('retirement refuses to signal a record whose owner it cannot recognize', as assert.equal(result.reason, 'owner-unproven', ownership); if (ownership === 'sideways') await assert.rejects( - stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), { - paths, - host: createTestHostEnvironment(), - }), + stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), paths), (error: unknown) => error instanceof AppError && error.details?.reason === 'daemon-owner-unrecognized', ); diff --git a/src/cli/commands/__tests__/daemon.test.ts b/src/cli/commands/__tests__/daemon.test.ts index 8536fe09db..e9e9bc50b1 100644 --- a/src/cli/commands/__tests__/daemon.test.ts +++ b/src/cli/commands/__tests__/daemon.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import { afterEach, expect, test, vi } from 'vitest'; import type { DaemonStopResult } from '@agent-device/daemon-contracts/daemon-stop'; +import { AppError } from '@agent-device/kernel/errors'; import { mkdtempForTestSync, bindTestCommandHost, @@ -38,7 +39,6 @@ vi.mock('@agent-device/daemon-contracts/daemon-registration', async (importOrigi vi.mock('../shared.ts', () => ({ writeCommandOutput: mocks.writeCommandOutput })); import { daemonCommand } from '../daemon.ts'; -import * as daemonClientStop from '../../../daemon-client/daemon-client-stop.ts'; bindTestCommandHost(); @@ -386,35 +386,42 @@ test('daemon status names a record whose owner it cannot recognize', async () => expect(render()).toContain('pid 321, owner-not-recognized'); }); -test('daemon stop of a host-owned record goes through the one router with the host the command runs on', async () => { +// A host-owned record reaches the central retirement through the one router, and at this layer the +// owner's rule is the contract: the refusal the retirement raises is what `daemon stop` reports, +// unchanged. What the real retirement raises for a live embedded owner, without signaling it, is +// pinned at that seam by src/__tests__/daemon-registration-embedded-owner.test.ts. +test('daemon stop routes a host-owned record through the one router and reports its refusal', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-embedded-'); fs.writeFileSync( `${stateDir}/daemon.json`, JSON.stringify({ pid: 555, processStartTime: 'start-time', ownership: 'embedded' }), { mode: 0o600 }, ); - vi.spyOn(daemonClientStop, 'stopRecordedDaemon').mockResolvedValue({ - ...GRACEFUL_RESULT, - warnings: ['1 in-flight request ignored cancellation and was abandoned.'], - }); - - const host = createTestHostEnvironment(); + mocks.stopDaemon.mockRejectedValue( + new AppError('COMMAND_FAILED', 'The daemon was not stopped, because its owner ends it.', { + reason: 'daemon-owner-embedded', + }), + ); try { - await daemonCommand({ - host, - positionals: ['stop'], - flags: { help: false, json: false, stateDir, version: false }, - client: {} as never, - }); + await expect( + daemonCommand({ + host: createTestHostEnvironment(), + positionals: ['stop'], + flags: { help: false, json: false, stateDir, version: false }, + client: {} as never, + }), + ).rejects.toMatchObject({ details: { reason: 'daemon-owner-embedded' } }); - // A host-owned runtime is never signaled, so it never reaches the retirement a detached stop uses. - expect(mocks.stopDaemon).not.toHaveBeenCalled(); - const render = mocks.writeCommandOutput.mock.calls[0]![2] as () => string; - expect(render()).toContain('Daemon stopped (graceful).'); - expect(render()).toContain('ignored cancellation'); + // The refusal travels with the lifetime and owner from the one read, and nothing was cleaned. + expect(mocks.stopDaemon).toHaveBeenCalledWith({ + paths: expect.objectContaining({ baseDir: stateDir }), + observed: { pid: 555, startTime: 'start-time', ownership: 'embedded' }, + }); + expect(mocks.cleanupDaemonOwner).not.toHaveBeenCalled(); + expect(mocks.writeCommandOutput).not.toHaveBeenCalled(); + expect(fs.readFileSync(`${stateDir}/daemon.json`, 'utf8')).toContain('"embedded"'); } finally { - vi.restoreAllMocks(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts index 56818291a0..b32b278c1e 100644 --- a/src/cli/commands/daemon.ts +++ b/src/cli/commands/daemon.ts @@ -46,7 +46,7 @@ export const daemonCommand: ClientCommandHandler = async ({ positionals, flags, if (flags.clean === true && record.ownership !== 'absent' && record.ownership !== 'detached') { throw daemonCleanRefusedForUnprovenOwner(paths, record.ownership); } - const result = await stopRecordedDaemon(record, { paths, host }); + const result = await stopRecordedDaemon(record, paths); const cleanupOwner = readStopCleanupOwner(record); const shouldClean = flags.clean === true && cleanupOwner !== null && result.stopped; if (shouldClean) { diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 49d3cbb045..33f90ca8c7 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -514,7 +514,7 @@ Runner and daemon lifecycle (applies to simulators too): Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the 60 s stall window, so a ceiling set below 60 s also ends a silent own launch this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. - A host-owned runtime records ownership and daemon status shows host-owned or owner-not-recognized. Managed clients reuse a matching host-owned runtime, but never replace, signal or retire it after a timeout. A host closes its own runtime directly. daemon stop refuses a live host-owned or unrecognized owner without signaling it; --clean also requires a proven detached owner. Detached stop and cleanup use the lifetime and ownership from one registration read. An unreadable daemon.json fails stop with daemon-record-unreadable. + A host-owned runtime records ownership and daemon status shows host-owned or owner-not-recognized. Managed clients reuse a matching host-owned runtime. Takeover, stop, drain, and timeout recovery all ask the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one so the state directory stays usable. A host closes its own live runtime directly. daemon stop refuses a live host-owned or unrecognized owner without signaling it; --clean also requires a proven detached owner. Detached stop and cleanup use the lifetime and ownership from one registration read. An unreadable daemon.json fails stop with daemon-record-unreadable. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts index 82bf05d402..a66e5d6d48 100644 --- a/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts @@ -83,6 +83,30 @@ async function startStaleRegistration(ownership?: 'embedded') { return { stateDir, paths, daemon, ...registered }; } +/** + * The same stale registration left behind by an owner that died: a real child publishes it and is then + * killed without retiring anything, so the record names a lifetime the process table can disprove. Its + * record and lock stay, which is the wedged state directory every managed command must get past. + */ +async function crashedStaleRegistration(ownership: 'embedded' | 'unrecognized' | 'detached') { + const { stateDir, paths } = stateDirAndPaths(); + const registered = await registerDaemon(paths, { + // Nothing answers this port: the point is that the client never asks the dead owner anything. + httpPort: 1, + version: '0.0.0-stale', + ...(ownership === 'embedded' ? { ownership: 'embedded' } : {}), + }); + if (ownership === 'unrecognized') { + const record = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')); + fs.writeFileSync(paths.infoPath, JSON.stringify({ ...record, ownership: 'sideways' })); + assert.equal(readDaemonInfo(paths.infoPath)?.ownership, 'unknown'); + } + process.kill(registered.child.pid, 'SIGKILL'); + await registered.child.exited; + assert.equal(isProcessAlive(registered.child.pid), false); + return { stateDir, paths, ...registered }; +} + for (const [ownership, reason, hint] of [ ['embedded', DAEMON_LIFECYCLE_REASONS.ownerEmbedded, /Close the host/], ['sideways', 'daemon-owner-unrecognized', /Update agent-device/], @@ -145,24 +169,52 @@ test('the same record with no host owner is still the takeover this client perfo } }); -test.each(['unknown', 'embedded', 'detached'] as const)( - 'timeout recovery respects %s ownership', - (ownership) => { - const { stateDir } = stateDirAndPaths(); - const settings = resolveClientSettings( - request(stateDir), - undefined, - createTestHostEnvironment(), - 'managed', - ); - assert.equal( - resolveTimeoutRecovery(settings, { - token: 'token', - pid: 4242, - processStartTime: 'start', - ownership, - }), - ownership === 'detached' ? 'retire-unresponsive' : 'none', - ); - }, -); +for (const ownership of ['embedded', 'unrecognized', 'detached'] as const) + test(`a managed command recovers the record a crashed ${ownership} owner left behind`, async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + // The wedged state directory: an incompatible-version record and its lock, naming a lifetime the + // process table disproves. Ownership is judged inside the retirement, so all three recover: the + // dead owner is proven gone, nothing is signaled, and the managed command starts a fresh daemon. + const { stateDir, paths, child } = await crashedStaleRegistration(ownership); + const fresh = await startScriptedDaemon(okResult({ devices: [] })); + launchesFreshDaemonAt(spawn, paths, fresh.port); + const killSpy = vi.spyOn(process, 'kill'); + + try { + const response = await createDaemonTransport(createTestHostEnvironment())(request(stateDir)); + + assert.equal(response.ok, true, JSON.stringify(response)); + assert.equal(spawn.mock.calls.length, 1, 'the state directory was taken over, not wedged'); + // The dead owner's record is gone: the fresh daemon's registration names a live process, and the + // dead lifetime received no signal — its exit was proved from the process table, not caused. + assert.notEqual(readDaemonInfo(paths.infoPath)?.pid, child.pid); + assert.deepEqual( + signalsSent(killSpy).filter(([target]) => target === child.pid), + [], + ); + } finally { + await closeLoopbackServer(fresh.server); + } + }); + +test('timeout recovery asks only whether this client may, and attach never may', () => { + const { stateDir } = stateDirAndPaths(); + const managed = resolveClientSettings( + request(stateDir), + undefined, + createTestHostEnvironment(), + 'managed', + ); + const attach = resolveClientSettings( + request(stateDir), + undefined, + createTestHostEnvironment(), + 'attach', + ); + // Recovery is the retirement's judgement, so the client decides only whether it may ask at all: an + // attach client never does, and every other lifecycle asks. The retirement answers for the record's + // owner — a live embedded or unrecognized one is refused without a signal, which + // daemon-registration-embedded-owner.test.ts pins at that seam for both modes. + assert.equal(resolveTimeoutRecovery(managed), 'retire-unresponsive'); + assert.equal(resolveTimeoutRecovery(attach), 'none'); +}); diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index f8df7d5076..875e855ea5 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -60,14 +60,15 @@ LOOPBACK_BLOCK_LIST.addSubnet('127.0.0.0', 8, 'ipv4'); LOOPBACK_BLOCK_LIST.addAddress('::1', 'ipv6'); LOOPBACK_BLOCK_LIST.addSubnet('::ffff:127.0.0.0', 104, 'ipv6'); +/** + * Whether a timed-out request may retire the daemon it was waiting on. An attach client never may: the + * daemon is not its to kill, and `ensureDaemon` is where an unresponsive one is replaced. Every other + * caller asks the retirement, which judges the record's owner before it signals anything. + */ export function resolveTimeoutRecovery( settings: DaemonClientSettings, - info: DaemonInfo, ): 'retire-unresponsive' | 'none' { if (settings.daemonLifecycle === 'attach') return 'none'; - // Only a record that proves a detached daemon authorizes a kill: an embedded one belongs to a host, - // and one whose owner this build cannot read says nothing about what is safe to signal. - if (info.ownership !== 'detached') return 'none'; return 'retire-unresponsive'; } diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index ebd6f7e6d4..2dfcbb8736 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -40,9 +40,7 @@ import { import { canConnect, isDaemonTransportUnavailableError } from './daemon-client-transport.ts'; import { daemonContendedError, - daemonNotRetirableError, - daemonRecordOwnerUnrecognizedError, - daemonRetirementUnconfirmedError, + daemonTakeoverRetirementError, daemonStopCommand, } from './daemon-lifecycle-errors.ts'; import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; @@ -152,22 +150,6 @@ export async function readReusableLocalDaemon( } if (!settings.retireIncompatibleContender) throw daemonContendedError(); - // A host owns an embedded runtime, so a managed client that wants this state directory asks the host - // for it instead of retiring the runtime that holds it. - if (existing.ownership === 'embedded') { - throw daemonNotRetirableError({ - info: existing, - stateDir: settings.paths.baseDir, - }); - } - // A record stating an owner this build cannot read is not a licence to replace what it names: the - // whole point of reading the owner is to know whether a pid may be signaled, and this says nothing. - if (existing.ownership === 'unknown') { - throw daemonRecordOwnerUnrecognizedError({ - info: existing, - stateDir: settings.paths.baseDir, - }); - } if (remainingStartupBudget(deadline) < MINIMUM_DAEMON_TAKEOVER_BUDGET_MS) return null; emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); await retireDaemonForTakeover(existing, settings.paths); @@ -226,7 +208,8 @@ export async function retireDaemonForTakeover( mode: 'graceful' | 'force' = 'graceful', ): Promise { const retirement = await stopDaemonForTakeover(existing, paths, mode); - if (retirement.status === 'retained') throw daemonRetirementUnconfirmedError(retirement, paths); + if (retirement.status === 'retained') + throw daemonTakeoverRetirementError(retirement, existing, paths); } /** A live pid is probed again before being judged unreachable; liveness is signal 0, not `ps`. */ diff --git a/src/daemon-client/daemon-client-stop.ts b/src/daemon-client/daemon-client-stop.ts index a1f69775e5..0d76637097 100644 --- a/src/daemon-client/daemon-client-stop.ts +++ b/src/daemon-client/daemon-client-stop.ts @@ -1,58 +1,43 @@ import { readDaemonShutdownReport } from '@agent-device/daemon-contracts/daemon-shutdown-report'; -import type { DaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { + observedDaemonIdentityOfRegistration, + type DaemonStopIdentity, +} from '@agent-device/daemon-contracts/daemon-registration'; import { stopDaemon, mergeDaemonShutdownReport, type DaemonStopResult, } from '@agent-device/daemon-contracts/daemon-stop'; import type { DaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { AppError } from '@agent-device/kernel/errors'; +/** + * Stops the daemon one registration read names. Ownership travels with the lifetime it read, and the + * central retirement judges it: an embedded runtime or an unrecognized owner comes back as the refusal + * that names its owner, with nothing signaled. + */ export async function stopRecordedDaemon( record: DaemonStopIdentity, - target: { paths: DaemonPaths; host: HostEnvironment }, + paths: DaemonPaths, ): Promise { if (record.ownership === 'unreadable') { throw new AppError( 'COMMAND_FAILED', - `The daemon record on ${target.paths.baseDir} cannot be read, so this stop cannot tell whether a daemon runs.`, + `The daemon record on ${paths.baseDir} cannot be read, so this stop cannot tell whether a daemon runs.`, { reason: 'daemon-record-unreadable', - stateDir: target.paths.baseDir, + stateDir: paths.baseDir, hint: 'Nothing was signaled. Inspect or remove that record, then retry.', }, ); } - const observed = - record.identity && - (record.ownership === 'embedded' || - record.ownership === 'unknown' || - record.ownership === 'detached') - ? { ...record.identity, ownership: record.ownership } - : null; - let stopped: DaemonStopResult; - try { - stopped = await stopDaemon({ paths: target.paths, observed }); - } catch (error) { - if ( - error instanceof AppError && - (error.details?.terminationReason === 'embedded-owner' || - error.details?.terminationReason === 'owner-unproven') - ) { - throw new AppError(error.code, error.message, { - ...error.details, - reason: - error.details.terminationReason === 'embedded-owner' - ? 'daemon-owner-embedded' - : 'daemon-owner-unrecognized', - hint: 'Nothing was signaled. Close the host that started this runtime, or repair its record after the process has exited.', - }); - } - throw error; - } + // The retirement judges the owner this one read stated; nothing here decides it. + const stopped = await stopDaemon({ + paths, + observed: observedDaemonIdentityOfRegistration(record), + }); return mergeDaemonShutdownReport( stopped, - stopped.mode === 'graceful' ? readDaemonShutdownReport(target.paths.baseDir) : null, + stopped.mode === 'graceful' ? readDaemonShutdownReport(paths.baseDir) : null, ); } diff --git a/src/daemon-client/daemon-client.ts b/src/daemon-client/daemon-client.ts index 2b09e75e64..8eca223f52 100644 --- a/src/daemon-client/daemon-client.ts +++ b/src/daemon-client/daemon-client.ts @@ -149,7 +149,7 @@ async function sendToDaemonOnHost( host, onProgress: options.onProgress, signal: options.signal, - timeoutRecovery: resolveTimeoutRecovery(settings, info), + timeoutRecovery: resolveTimeoutRecovery(settings), }, ), { requestId, command: req.command }, diff --git a/src/daemon-client/daemon-lifecycle-errors.ts b/src/daemon-client/daemon-lifecycle-errors.ts index 54bde7aa6f..09e2913bbb 100644 --- a/src/daemon-client/daemon-lifecycle-errors.ts +++ b/src/daemon-client/daemon-lifecycle-errors.ts @@ -85,6 +85,24 @@ export function daemonRetirementUnconfirmedError( }); } +/** + * The refusal a takeover reports when its retirement kept the state, keyed on the reason that retirement + * gives for itself. Ownership is judged inside the retirement, so this is the one place that answer + * becomes a client error: an owner's rule keeps the reason every other surface uses for that owner, and + * anything else stays a replacement this client could not confirm. + */ +export function daemonTakeoverRetirementError( + retirement: Extract, + info: DaemonInfo, + paths: DaemonPaths, +): AppError { + if (retirement.reason === 'embedded-owner') + return daemonNotRetirableError({ info, stateDir: paths.baseDir }); + if (retirement.reason === 'owner-unproven') + return daemonRecordOwnerUnrecognizedError({ info, stateDir: paths.baseDir }); + return daemonRetirementUnconfirmedError(retirement, paths); +} + export function isDaemonNotRunningError(error: unknown): boolean { return ( error instanceof AppError && error.details?.reason === DAEMON_UNAVAILABLE_REASONS.notRunning @@ -141,7 +159,7 @@ export function daemonBusyError( } /** Refuses managed replacement when the record cannot prove a detached owner. */ -export function daemonRecordOwnerUnrecognizedError(params: { +function daemonRecordOwnerUnrecognizedError(params: { info: DaemonInfo; stateDir: string; }): AppError { diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 50b3a54759..59b11285f3 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -90,7 +90,7 @@ if ('cloudArtifacts' in result) { A client talks to one local daemon per state directory. `daemonLifecycle` says who owns that daemon. -A host-owned runtime records its ownership. A managed client reuses it while it matches, but never replaces, signals or retires it after a timeout. The host closes its own runtime directly. `daemon stop` refuses a live host-owned or unrecognized owner without signaling it; `--clean` also requires a proven detached owner. An unreadable `daemon.json` fails stop with `daemon-record-unreadable`. +A host-owned runtime records its ownership. A managed client reuses it while it matches. Every takeover, stop, drain, and timeout recovery asks the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one, so the state directory stays usable. The host closes its live runtime directly. `daemon stop` refuses a live host-owned or unrecognized owner without signaling it; `--clean` also requires a proven detached owner. An unreadable `daemon.json` fails stop with `daemon-record-unreadable`. | Mode | What the client does with the daemon | | --- | --- | From e574ba365abbce8766bbed17ff2bae48ae8dd0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 15:41:21 +0200 Subject: [PATCH 4/4] docs(daemon): name the reason a --clean stop refuses an unreadable record daemon stop --clean checks the record's owner before the stop route reads it, so an unreadable record fails with daemon-clean-not-detached, not daemon-record-unreadable. The help and docs said the latter unconditionally; qualify it with the flag that decides it. https://github.com/callstack/agent-device/pull/3806#issuecomment-6262194144 --- src/commands/schema/cli-help.ts | 2 +- website/docs/docs/client-api.md | 2 +- website/docs/docs/commands.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 33f90ca8c7..d5e247aeaa 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -514,7 +514,7 @@ Runner and daemon lifecycle (applies to simulators too): Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the 60 s stall window, so a ceiling set below 60 s also ends a silent own launch this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. - A host-owned runtime records ownership and daemon status shows host-owned or owner-not-recognized. Managed clients reuse a matching host-owned runtime. Takeover, stop, drain, and timeout recovery all ask the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one so the state directory stays usable. A host closes its own live runtime directly. daemon stop refuses a live host-owned or unrecognized owner without signaling it; --clean also requires a proven detached owner. Detached stop and cleanup use the lifetime and ownership from one registration read. An unreadable daemon.json fails stop with daemon-record-unreadable. + A host-owned runtime records ownership and daemon status shows host-owned or owner-not-recognized. Managed clients reuse a matching host-owned runtime. Takeover, stop, drain, and timeout recovery all ask the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one so the state directory stays usable. A host closes its own live runtime directly. daemon stop refuses a live host-owned or unrecognized owner without signaling it; --clean also requires a proven detached owner and refuses host-owned, unrecognized and unreadable records with daemon-clean-not-detached. Detached stop and cleanup use the lifetime and ownership from one registration read. Without --clean, an unreadable daemon.json fails stop with daemon-record-unreadable. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 59b11285f3..69d33b3f45 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -90,7 +90,7 @@ if ('cloudArtifacts' in result) { A client talks to one local daemon per state directory. `daemonLifecycle` says who owns that daemon. -A host-owned runtime records its ownership. A managed client reuses it while it matches. Every takeover, stop, drain, and timeout recovery asks the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one, so the state directory stays usable. The host closes its live runtime directly. `daemon stop` refuses a live host-owned or unrecognized owner without signaling it; `--clean` also requires a proven detached owner. An unreadable `daemon.json` fails stop with `daemon-record-unreadable`. +A host-owned runtime records its ownership. A managed client reuses it while it matches. Every takeover, stop, drain, and timeout recovery asks the central retirement, which judges the record's owner once: a live host-owned or unrecognized owner is refused without a signal, while a record whose owner provably exited is retired like any abandoned one, so the state directory stays usable. The host closes its live runtime directly. `daemon stop` refuses a live host-owned or unrecognized owner without signaling it; `--clean` also requires a proven detached owner and refuses host-owned, unrecognized and unreadable records with `daemon-clean-not-detached`. Without `--clean`, an unreadable `daemon.json` fails stop with `daemon-record-unreadable`. | Mode | What the client does with the daemon | | --- | --- | diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 59f46b983e..74e5533c7e 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -85,7 +85,7 @@ Device claims and the daemon: - `device status` lists host-local device claims without starting or contacting a daemon; `--stale` shows the stale claims it hides by default (owner exited, PID reused, state directory deleted, or daemon replaced), some of whose owners may still be running. Scope it with `--platform` plus `--udid` (Apple) or `--serial` (Android). A claim held by another live or uncertain owner blocks `open`, unless the device rebooted after the claim was taken: then `open` settles that owner's durable resources, takes the claim, and reports the release in `warnings`. - `device release --stale` settles the durable resources of an owner that provably can't release its claim (its process exited or a newer daemon replaced it), then clears the claim. Live, uncertain, PID-reused, corrupt, and deleted-state-directory claims are refused with the reason. See [Sessions](/docs/sessions#release-a-device-another-session-or-workspace-holds). - A command that needs the shared local daemon starts it for its state directory when none runs; `device status` and `device release --stale` never contact one. When the running daemon comes from another version or build, the command replaces it only if it reports no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise the command fails with `daemon-busy` and leaves the daemon alone. A daemon from a release that predates this check is never replaced, and fails with `daemon-incompatible`. End a busy or older daemon deliberately with `daemon stop`. Node.js clients choose this per client; see [Daemon lifecycle](/docs/client-api#daemon-lifecycle). -- `daemon stop --state-dir ` stops a detached local daemon after verifying its identity. Stop and `--clean` use the lifetime and ownership from one registration read. A live host-owned or unrecognized owner is refused without signaling; its host closes the runtime directly. An unreadable `daemon.json` fails stop with `daemon-record-unreadable`. `daemon status` shows `host-owned` or `owner-not-recognized`. `--clean` also removes the Apple runner processes and leases that daemon retained. +- `daemon stop --state-dir ` stops a detached local daemon after verifying its identity. Stop and `--clean` use the lifetime and ownership from one registration read. A live host-owned or unrecognized owner is refused without signaling; its host closes the runtime directly. Without `--clean`, an unreadable `daemon.json` fails stop with `daemon-record-unreadable`; with `--clean`, host-owned, unrecognized and unreadable records fail with `daemon-clean-not-detached`. `daemon status` shows `host-owned` or `owner-not-recognized`. `--clean` also removes the Apple runner processes and leases that daemon retained. - `daemon status [--state-dir ] [--json]` reads the daemon's record without contacting it. For a running daemon it lists every provider it registered, built in or installed, and the reason any plugin failed to register. See [provider plugins](/docs/plugins). Open apps and links: