fix(daemon): replace fixed 15s startup kill with progress-aware startup budget

.github/workflows/ios.yml

@@ -169,6 +169,14 @@ jobs:
       - name: Preflight iOS runner through public CLI
         run: |
           pnpm clean:daemon
+          # The public-SDK record is the cold-startup evidence; the CLI timer only wraps attach.
+          node --input-type=module -e '
+            const { performance } = await import("node:perf_hooks");
+            const { ensureDaemon } = await import("./dist/src/index.js");
+            const t0 = performance.now();
+            const c = await ensureDaemon({ stateDir: process.env.AGENT_DEVICE_STATE_DIR });
+            console.log(JSON.stringify({ phase: "daemon_cold_startup", started: c.started, pid: c.daemon.pid, durationMs: Math.round(performance.now() - t0) }));
+          '
           node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json
 
       # The replay reuses the prepared runner in the same daemon. --debug keeps every step's request

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - The PNG worker no longer calls `process.emitWarning` when it is unavailable, and it starts with an empty environment (`env: {}`). The daemon registers a sink when it pre-warms the worker, so every transition to unavailable (spawn failure, missing entry, load error, exit) leaves exactly one `png_worker_unavailable` event, with a typed `reason` (`entry-not-found`, `spawn-failed`, `worker-error`, `worker-exited`) and its `detail`, in `daemon.log`, with or without `--debug`. Requests then continue on the in-process fallback without repeating the event.
 - Behavior change: `resolveHubAppReference`, shipped through `agent-device/plugins/webdriver`, now needs `cwd` to resolve a relative app path. Without it the call fails with `INVALID_ARGS` and `details.reason: provider_app_cwd_missing`; it used to resolve against the process's working directory. Absolute paths, URLs and provider references are unaffected. The daemon fills it from the client's working directory (or its host's) for every provider lease request, so `connect` and `lease_allocate` are unchanged.
+- A client starting a local daemon now waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower instead of failing it. The wait stops at a ceiling of 120 s by default; set `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` to a positive integer number of milliseconds to override it for the CLI, MCP, and Node clients alike. The variable is read only when the client must start a daemon; a value that is not a positive integer fails with `INVALID_ARGS` before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A failed startup names the outcome in `error.details.reason`, and you should branch on it, never on the message: `startup-exited` (the process ended before registering), `startup-stalled` (alive but no startup progress for a fixed 60 s window, which the ceiling variable does not change; the daemon is stopped), or `startup-ceiling` (a startup was still live — this client's daemon or another client's — when the ceiling passed; the client stops its own launch and never signals another client's live daemon). `details.phases` carries the startup phase timings the client observed and `details.startupTimeoutMs` the effective ceiling.
 - The `daemon-not-running` hint now always names `--state-dir <dir>`, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`.
 - Fix `ensureDaemon()`, the CLI and MCP killing a daemon they had just asked to leave. After the daemon answers the quiesce request with `draining`, the client now waits up to 60 s for it to exit by itself and clean up its registration, so plugin disposal and lease finalization finish (on Windows a signal ends the process outright, and on POSIX a drain longer than 3 s was killed). A daemon still running at the deadline that fails its liveness probes is force-retired; one that still answers is left running and the call fails with `details.reason: 'daemon-drain-timeout'`, with a hint naming `agent-device daemon stop --state-dir <path>`.
 - `@agent-device/testmu` is now a version 2 provider plugin. `import testmu from '@agent-device/testmu'` gives a factory, `testmu({ credentials: { username, accessKey } })`, that reads those credentials instead of `LT_USERNAME` and `LT_ACCESS_KEY`; without options it reads the variables as before. The package still declares the version 1 plugin entry, so `agent-device plugins add @agent-device/testmu` keeps working on hosts that predate the version 2 contract.

packages/daemon-contracts/package.json

@@ -44,6 +44,10 @@
       "types": "./src/daemon-shutdown-report.ts",
       "default": "./src/daemon-shutdown-report.ts"
     },
+    "./daemon-startup-phases": {
+      "types": "./src/daemon-startup-phases.ts",
+      "default": "./src/daemon-startup-phases.ts"
+    },
     "./daemon-stop": {
       "types": "./src/daemon-stop.ts",
       "default": "./src/daemon-stop.ts"

packages/daemon-contracts/src/daemon-registration-owner.ts

@@ -44,6 +44,7 @@ import {
   resolveDaemonShutdownReportPath,
   type DaemonShutdownOutcome,
 } from './daemon-shutdown-report.ts';
+import { sweepDeadDaemonStartupPhasesFiles } from './daemon-startup-phases.ts';
 
 export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 });
 export type DaemonRegistrationFields = Readonly<{
@@ -85,6 +86,7 @@ export async function tryAcquireDaemonRegistration(
   try {
     acquisition.assertHeld();
     fs.rmSync(resolveDaemonShutdownReportPath(boundPaths.baseDir), { force: true });
+    sweepDeadDaemonStartupPhasesFiles(boundPaths.baseDir);
   } catch (error) {
     await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath);
   }

packages/daemon-contracts/src/daemon-startup-phases.test.ts

@@ -0,0 +1,144 @@
+import { appendFile, readFile, writeFile } from 'node:fs/promises';
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { expect, test } from 'vitest';
+import { tryAcquireDaemonRegistration } from '@agent-device/daemon-contracts/daemon-registration-owner';
+import {
+  readDaemonStartupPhaseMarkers,
+  recordDaemonStartupMarker,
+  removeDaemonStartupPhasesFile,
+  resolveDaemonStartupPhasesPath,
+} from '@agent-device/daemon-contracts/daemon-startup-phases';
+import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution';
+import { mkdtempForTest } from '@agent-device/test-kit';
+
+test('per-launch marker files: a fresh launch never reads an older launch phases', async () => {
+  const stateDir = await mkdtempForTest('agent-device-markers-');
+  const first = resolveDaemonStartupPhasesPath(stateDir, 1111);
+  const second = resolveDaemonStartupPhasesPath(stateDir, 2222);
+  recordDaemonStartupMarker(first, 'lock-acquired');
+  expect(second).not.toBe(first);
+  expect(existsSync(second)).toBe(false);
+  recordDaemonStartupMarker(second, 'modules-loaded');
+  const firstPhases = readDaemonStartupPhaseMarkers(readFileSync(first, 'utf8'));
+  const secondPhases = readDaemonStartupPhaseMarkers(readFileSync(second, 'utf8'));
+  expect(Object.keys(firstPhases)).toEqual(['lock-acquired']);
+  expect(Object.keys(secondPhases)).toEqual(['modules-loaded']);
+});
+
+test('marker reading keeps the earliest record when a phase replays after a later one', () => {
+  const text = [
+    '{"data":{"startupPhase":"listening","timestampMs":500}}',
+    '{"data":{"startupPhase":"lock-acquired","timestampMs":20}}',
+    '{"data":{"startupPhase":"lock-acquired","timestampMs":35}}',
+  ].join('\n');
+  const phases = readDaemonStartupPhaseMarkers(text);
+  expect(phases['lock-acquired']).toBe(20);
+  expect(phases.listening).toBe(500);
+});
+
+test('marker reading ignores unknown phases and unparsable lines', () => {
+  const text = [
+    '{"data":{"startupPhase":"heartbeat","timestampMs":10}}',
+    '{"data":{"startupPhase":"registered","timestampMs":"7"}}',
+    'not json',
+    '{"data":{"startupPhase":"listening","timestampMs":30}}',
+  ].join('\n');
+  expect(readDaemonStartupPhaseMarkers(text)).toEqual({ listening: 30 });
+});
+
+test('the producer lifetime claim empties a same-pid leftover', async () => {
+  // Pid reuse after both the old client and daemon died before either cleanup: the entry removes
+  // its record before its first marker; this child repeats that producer sequence itself.
+  const stateDir = await mkdtempForTest('agent-device-markers-');
+  writeFileSync(path.join(stateDir, 'package.json'), '{"type":"module"}');
+  const entry = path.join(stateDir, 'claim-producer.js');
+  const registrationUrl = new URL('./daemon-startup-phases.ts', import.meta.url).href;
+  writeFileSync(
+    entry,
+    `import { removeDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)};
+const file = resolveDaemonStartupPhasesPath(${JSON.stringify(stateDir)}, process.pid);
+recordDaemonStartupMarker(file, 'listening');
+recordDaemonStartupMarker(file, 'registered');
+removeDaemonStartupPhasesFile(file);
+recordDaemonStartupMarker(file, 'modules-loaded');
+`,
+  );
+  const child = spawnSync(process.execPath, ['--experimental-strip-types', entry], {
+    encoding: 'utf8',
+  });
+  if (child.status !== 0) throw new Error(child.stderr);
+  const phases = readDaemonStartupPhaseMarkers(
+    readFileSync(resolveDaemonStartupPhasesPath(stateDir, child.pid!), 'utf8'),
+  );
+  expect(phases).toEqual({ 'modules-loaded': expect.any(Number) });
+});
+
+test.each([
+  ['remove', (p: string, cb: (e: unknown) => void) => removeDaemonStartupPhasesFile(p, cb)],
+  [
+    'record',
+    (p: string, cb: (e: unknown) => void) => recordDaemonStartupMarker(p, 'modules-loaded', cb),
+  ],
+] as const)('a refused %s reports through the callback instead of throwing', async (_name, run) => {
+  const stateDir = await mkdtempForTest('agent-device-markers-');
+  const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777);
+  mkdirSync(markerPath, { recursive: true });
+  const reported: unknown[] = [];
+  expect(() =>
+    run(markerPath, (error) => {
+      reported.push(error);
+    }),
+  ).not.toThrow();
+  expect(reported).toHaveLength(1);
+  expect(reported[0]).toHaveProperty('message');
+});
+
+test('acquiring the registration retires files of dead or unowned pids only', async () => {
+  const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-'));
+  const dead = resolveDaemonStartupPhasesPath(paths.baseDir, spawnDeadPid());
+  const own = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid);
+  const unparseable = path.join(paths.baseDir, 'daemon-startup-phases-.jsonl');
+  const legacy = path.join(paths.baseDir, 'daemon-startup-phases.jsonl');
+  await Promise.all([
+    writeFile(own, 'x\n', 'utf8'),
+    writeFile(dead, 'x\n', 'utf8'),
+    writeFile(unparseable, 'x\n', 'utf8'),
+    writeFile(legacy, 'x\n', 'utf8'),
+  ]);
+  const attempt = await tryAcquireDaemonRegistration(paths);
+  expect(attempt.status).toBe('acquired');
+  if (attempt.status !== 'acquired') throw new Error('registration refused');
+  expect(existsSync(dead)).toBe(false);
+  expect(existsSync(unparseable)).toBe(false);
+  expect(existsSync(own)).toBe(true);
+  expect(existsSync(legacy)).toBe(true);
+  await attempt.owner.finish();
+});
+
+test('removeDaemonStartupPhasesFile removes a file and tolerates one already gone', async () => {
+  const stateDir = await mkdtempForTest('agent-device-markers-');
+  const file = resolveDaemonStartupPhasesPath(stateDir, 4242);
+  await writeFile(file, 'x\n', 'utf8');
+  removeDaemonStartupPhasesFile(file);
+  expect(existsSync(file)).toBe(false);
+  expect(() => removeDaemonStartupPhasesFile(file)).not.toThrow();
+});
+
+test('a crash-replay append keeps the file valid JSONL', async () => {
+  const stateDir = await mkdtempForTest('agent-device-markers-');
+  const markerPath = resolveDaemonStartupPhasesPath(stateDir, 999);
+  await appendFile(markerPath, '{"data":{"startupPhase":"modules-lo', 'utf8');
+  recordDaemonStartupMarker(markerPath, 'composition-loaded');
+  const tail = await readFile(markerPath, 'utf8');
+  expect(Object.keys(readDaemonStartupPhaseMarkers(tail))).toEqual(['composition-loaded']);
+});
+
+function spawnDeadPid(): number {
+  const child = spawnSync(process.execPath, ['-e', ''], { encoding: 'utf8' });
+  if (child.error) throw child.error;
+  const pid = child.pid;
+  if (pid === undefined) throw new Error('spawnSync returned no pid');
+  return pid;
+}

packages/daemon-contracts/src/daemon-startup-phases.ts

@@ -0,0 +1,137 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors';
+import { isProcessAlive } from '@agent-device/host-kit/process';
+
+/**
+ * Startup phase markers: a per-launch record, named for the writing pid, of how far that startup
+ * got, so a waiting client tells "slow but progressing" from "stuck" without a heartbeat. The
+ * writer removes it on completion, the client after a joined exit, and each new registration
+ * owner sweeps the files of provably dead pids.
+ */
+const DAEMON_STARTUP_PHASES = Object.freeze([
+  'modules-loaded',
+  'composition-loaded',
+  'lock-acquired',
+  'recovery-finished',
+  'listening',
+  'registered',
+] as const);
+
+export type DaemonStartupPhase = (typeof DAEMON_STARTUP_PHASES)[number];
+
+const DAEMON_STARTUP_PHASE_SET: ReadonlySet<string> = new Set<string>(DAEMON_STARTUP_PHASES);
+
+const DAEMON_STARTUP_PHASES_ENTRY = /^daemon-startup-phases-(\d*)\.jsonl$/;
+
+/** The marker file the daemon process `pid` writes in `stateDir`; an absent file is no evidence. */
+export function resolveDaemonStartupPhasesPath(stateDir: string, pid: number): string {
+  return path.join(stateDir, `daemon-startup-phases-${pid}.jsonl`);
+}
+
+/** Appends one phase marker; a refused append goes to `reportFailure`, never fatal to a startup. */
+export function recordDaemonStartupMarker(
+  startupPhasesPath: string,
+  phase: DaemonStartupPhase,
+  reportFailure?: (error: NormalizedError) => void,
+): void {
+  try {
+    const timestampMs = Date.now();
+    fs.mkdirSync(path.dirname(startupPhasesPath), { recursive: true });
+    // A crash mid-append can leave no trailing newline; glue two appends and both are lost.
+    const separator = endsWithoutNewline(startupPhasesPath) ? '\n' : '';
+    fs.appendFileSync(
+      startupPhasesPath,
+      `${separator}${JSON.stringify({ data: { startupPhase: phase, timestampMs } })}\n`,
+    );
+  } catch (error) {
+    reportFailure?.(normalizeError(error));
+  }
+}
+
+function endsWithoutNewline(file: string): boolean {
+  let fd: number | undefined;
+  try {
+    const stats = fs.statSync(file);
+    if (stats.size === 0) return false;
+    const buffer = Buffer.alloc(1);
+    fd = fs.openSync(file, 'r');
+    if (fs.readSync(fd, buffer, 0, 1, stats.size - 1) !== 1) return false;
+    return buffer[0] !== 0x0a;
+  } catch {
+    return false;
+  } finally {
+    if (fd !== undefined) fs.closeSync(fd);
+  }
+}
+
+/**
+ * Removes this launch's record: before its first marker (a per-pid name identifies a writer, not
+ * a lifetime, so the entry claims its file first), and after a finished startup or joined exit.
+ * A refused removal only reports through `reportFailure`; it never stops a startup.
+ */
+export function removeDaemonStartupPhasesFile(
+  startupPhasesPath: string,
+  reportFailure?: (error: NormalizedError) => void,
+): void {
+  try {
+    fs.rmSync(startupPhasesPath, { force: true });
+  } catch (error) {
+    reportFailure?.(normalizeError(error));
+  }
+}
+
+/** Sweeps marker files of provably dead (or unnameable) pids; a live pid's file is kept. */
+export function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void {
+  let entries: string[];
+  try {
+    entries = fs.readdirSync(stateDir);
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const match = DAEMON_STARTUP_PHASES_ENTRY.exec(entry);
+    if (match === null) continue;
+    const deadPid = Number(match[1]);
+    if (deadPid === process.pid) continue;
+    if (!Number.isSafeInteger(deadPid) || deadPid <= 0 || !isProcessAlive(deadPid)) {
+      removeDaemonStartupPhasesFile(path.join(stateDir, entry));
+    }
+  }
+}
+
+/**
+ * The phases observed in a marker tail, keeping each phase's earliest occurrence. Unknown and
+ * unparsable lines are ignored: a client never fails a startup over a line it does not know.
+ */
+export function readDaemonStartupPhaseMarkers(
+  markerTail: string,
+): Partial<Record<DaemonStartupPhase, number>> {
+  const phases: Partial<Record<DaemonStartupPhase, number>> = {};
+  for (const line of markerTail.split('\n')) {
+    if (!line.includes('"startupPhase"')) continue;
+    let record: DaemonStartupMarkerRecord;
+    try {
+      record = JSON.parse(line) as DaemonStartupMarkerRecord;
+    } catch {
+      continue;
+    }
+    const phase = record.data?.startupPhase;
+    const timestampMs = record.data?.timestampMs;
+    if (
+      typeof phase !== 'string' ||
+      !DAEMON_STARTUP_PHASE_SET.has(phase) ||
+      typeof timestampMs !== 'number' ||
+      !Number.isFinite(timestampMs)
+    ) {
+      continue;
+    }
+    const seen = phases[phase as DaemonStartupPhase];
+    if (seen === undefined || timestampMs < seen) phases[phase as DaemonStartupPhase] = timestampMs;
+  }
+  return phases;
+}
+
+type DaemonStartupMarkerRecord = {
+  data?: { startupPhase?: unknown; timestampMs?: unknown };
+};

packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts

@@ -10,7 +10,7 @@ import type { ResourceDiagnostic } from '@agent-device/host-kit/diagnostics';
 import { createDurableResourceEnvelope } from '@agent-device/capture-kit';
 import { createTestAppLogLiveHandle, mkdtempForTestSync } from '@agent-device/test-kit';
 import { makeSessionStore } from '../../__tests__/support/store-factory.ts';
-import { flushDaemonStartupDiagnostics } from '../http/daemon-runtime.ts';
+import { flushDaemonStartupDiagnostics } from '../http/daemon-startup-diagnostics.ts';
 import type { SessionState } from '../../core/session-state.ts';
 import { unavailableDeviceRuntimeGateway } from '../../__tests__/support/unavailable-device-runtime-gateway.ts';