From 05be6bf3d2f7a0daf110aa21081030cc5c264f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 22:47:23 +0200 Subject: [PATCH 01/11] feat(daemon): the runtime reports its shutdown instead of exiting the process startDaemonRuntime no longer calls process.exit or installs process handlers; src/daemon.ts owns signals, fatal errors and the exit code. close() is idempotent and resolves to a ShutdownReport. A runtime that fails during startup rolls back what it acquired. Co-Authored-By: Claude Sonnet 5.5 --- .../src/application-lifecycle-runtime.ts | 10 +- .../support/daemon-runtime-deps-fixture.ts | 4 +- packages/daemon/src/index.ts | 3 +- .../__tests__/daemon-runtime-app-log.test.ts | 5 +- .../http/daemon-lease-finalizer.test.ts | 9 +- .../http/daemon-runtime-artifacts.test.ts | 4 +- .../server/http/daemon-runtime-close.test.ts | 95 +++++ .../http/daemon-runtime-composition.test.ts | 56 +-- .../http/daemon-runtime-idle-reap.test.ts | 57 ++- .../http/daemon-runtime-lease-expiry.test.ts | 8 +- .../daemon-runtime-lifecycle-shutdown.test.ts | 44 ++- .../daemon-runtime-metadata-ownership.test.ts | 41 +- .../daemon-runtime-owner-lifecycle.test.ts | 6 +- .../daemon-runtime-recording-reaper.test.ts | 4 +- .../daemon-runtime-rpc-flow-failure.test.ts | 13 +- .../daemon-runtime-snapshot-shutdown.test.ts | 2 +- .../daemon/src/server/http/daemon-runtime.ts | 352 ++++++++++-------- .../daemon/src/server/http/server-shutdown.ts | 62 +++ .../src/server/provider-lease-expiry.ts | 18 +- .../src/runner/runner-adoption.ts | 8 +- .../src/runner/runner-session.ts | 7 +- .../test-utils/public-daemon-runtime.ts | 6 +- src/daemon-composition-credentials.test.ts | 8 +- src/daemon-composition.test.ts | 4 +- src/daemon-entrypoint.test.ts | 34 +- src/daemon.ts | 45 ++- ...latform-runtime-apple-application-tools.ts | 4 +- ...form-runtime-application-resources.test.ts | 4 +- src/platform-runtime-gateway.ts | 8 +- src/provider-device-runtimes.test.ts | 4 +- 30 files changed, 536 insertions(+), 389 deletions(-) create mode 100644 packages/daemon/src/server/http/daemon-runtime-close.test.ts diff --git a/packages/contracts/src/application-lifecycle-runtime.ts b/packages/contracts/src/application-lifecycle-runtime.ts index 8db2fb2807..cf62df9d4d 100644 --- a/packages/contracts/src/application-lifecycle-runtime.ts +++ b/packages/contracts/src/application-lifecycle-runtime.ts @@ -382,8 +382,10 @@ export type AppleApplicationTools = Readonly<{ applyRuntimeHints(device: DeviceInfo, input: RuntimeHintsApplicationInput): Promise; clearRuntimeHints(device: DeviceInfo, input: RuntimeHintsApplicationInput): Promise; dismissCloseAlerts(device: DeviceInfo, input: CloseApplicationFinalizationInput): Promise; - detachRunnerSessionsForShutdown(): Promise; - finalizeRunnerSessionsForShutdown(): Promise; + /** The devices whose runner was handed to the next daemon instead of stopped. */ + detachRunnerSessionsForShutdown(): Promise; + /** The devices whose runner this daemon stopped. */ + finalizeRunnerSessionsForShutdown(): Promise; }>; /** Lazy Android mechanics used by the Android package's own lifecycle binding. */ @@ -413,6 +415,6 @@ export type AndroidApplicationTools = Readonly<{ /** Gateway-coordinated durable phases whose concrete mechanics stay package-owned. */ export type ApplicationLifecycleResourceLifecycle = Readonly<{ recoverStartupResources(input: Readonly<{ stateDir: string }>): Promise; - detachForDaemonShutdown(): Promise; - finalizeDaemonShutdown(): Promise; + detachForDaemonShutdown(): Promise; + finalizeDaemonShutdown(): Promise; }>; diff --git a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts index c4eaed88ed..ea50b0b8e7 100644 --- a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts +++ b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts @@ -25,8 +25,8 @@ export function applicationLifecycleResourceFixture( ): ApplicationLifecycleResourceLifecycle { return Object.freeze({ recoverStartupResources: vi.fn(async () => {}), - detachForDaemonShutdown: vi.fn(async () => {}), - finalizeDaemonShutdown: vi.fn(async () => {}), + detachForDaemonShutdown: vi.fn(async (): Promise => []), + finalizeDaemonShutdown: vi.fn(async (): Promise => []), ...overrides, }); } diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index b7ef46715a..b3204cc2cd 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -1,4 +1,5 @@ -export { startDaemonRuntime } from './server/http/daemon-runtime.ts'; +export { daemonStartupExitCode, startDaemonRuntime } from './server/http/daemon-runtime.ts'; +export type { ShutdownReport } from './server/http/server-shutdown.ts'; export type { ComposeDaemonRuntimeDeps, DaemonRuntimeController, diff --git a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts index 532ae85cde..5cea76642e 100644 --- a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts +++ b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts @@ -36,13 +36,12 @@ test('daemon startup configures the Apple runner owner after acquiring the lock, const source = fs.readFileSync(new URL('../http/daemon-runtime.ts', import.meta.url), 'utf8'); const acquiredLock = source.indexOf("if (acquisition.status !== 'acquired')"); const runnerOwnerConfigured = source.indexOf('await deps.ownerLifecycle.configureForDaemonLock('); - const lockFailureExit = source.indexOf('exit(', acquiredLock); + const lockFailureExit = source.indexOf("reason: 'daemon_registration_refused'", acquiredLock); expect(acquiredLock).toBeGreaterThanOrEqual(0); expect(lockFailureExit).toBeGreaterThan(acquiredLock); expect(runnerOwnerConfigured).toBeGreaterThan(acquiredLock); - // The configure call sits inside the post-lock try block, after the failure branch that exits - // for a lock held by another process. + // The configure call comes after the failure branch that refuses a lock held by another process. expect(runnerOwnerConfigured).toBeGreaterThan(lockFailureExit); }); diff --git a/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts b/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts index 5be1e1985a..0d56c8ec56 100644 --- a/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts +++ b/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts @@ -42,6 +42,7 @@ test('journals and bounds a hung recoverable lease release before the final drai await vi.advanceTimersByTimeAsync(10); await expect(drain).resolves.toEqual({ pending: [lease], + unknown: [lease], released: [], }); } finally { @@ -83,7 +84,7 @@ test('final drain joins a release that completes after the lease timeout', async const drain = expiredProviderLeaseReleaser.drain(2_000); await vi.advanceTimersByTimeAsync(500); - await expect(drain).resolves.toEqual({ pending: [], released: [lease] }); + await expect(drain).resolves.toEqual({ pending: [], unknown: [], released: [lease] }); } finally { expiredProviderLeaseReleaser.shutdown(); vi.useRealTimers(); @@ -129,7 +130,11 @@ test('a hung provider release does not starve another lease during shutdown', as expect(leaseRegistry.listActiveLeases()).toEqual([]); const drain = expiredProviderLeaseReleaser.drain(2_000); await vi.advanceTimersByTimeAsync(2_000); - await expect(drain).resolves.toEqual({ pending: [hungLease], released: [releasedLease] }); + await expect(drain).resolves.toEqual({ + pending: [hungLease], + unknown: [hungLease], + released: [releasedLease], + }); } finally { expiredProviderLeaseReleaser.shutdown(); vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts index 8e31d53d8d..b7e1f1e26a 100644 --- a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts @@ -28,8 +28,6 @@ async function downloadTrackedArtifact(env: Record, awaitConsump ...env, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture(), @@ -48,7 +46,7 @@ async function downloadTrackedArtifact(env: Record, awaitConsump } return { status: response.status, body, onDisk: fs.existsSync(artifactPath) }; } finally { - await runtime?.shutdown(); + await runtime?.close(); cleanupDownloadableArtifact(artifactId); } } diff --git a/packages/daemon/src/server/http/daemon-runtime-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-close.test.ts new file mode 100644 index 0000000000..480f80df5e --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-close.test.ts @@ -0,0 +1,95 @@ +import { expect, test, vi } from 'vitest'; +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { + applicationLifecycleResourceFixture, + daemonRuntimeDepsFixture, + platformOwnerLifecycleFixture, +} from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +function hostFor(stateDir: string) { + return createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }); +} + +function providerRuntime(provider: string, shutdown: () => Promise): ProviderDeviceRuntime { + return { provider, shutdown, leaseLifecycle: {} } as unknown as ProviderDeviceRuntime; +} + +const silent = { stderr: { write: () => {} }, stdout: { write: () => {} } }; + +test('close is idempotent and every caller receives the one report', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-close-'); + const stop = vi.fn(async () => {}); + const failing = vi.fn(async () => { + throw new Error('provider refused to stop'); + }); + const signalListeners = process.listenerCount('SIGTERM'); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + providerDeviceRuntimes: [providerRuntime('stops', stop), providerRuntime('fails', failing)], + deviceRuntimeGateway: Object.freeze({ + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture({ + detachForDaemonShutdown: async () => ['handed-off-device'], + finalizeDaemonShutdown: async () => ['stopped-device'], + }), + }), + }), + }); + expect(process.listenerCount('SIGTERM')).toBe(signalListeners); + + const [first, second] = await Promise.all([runtime.close(), runtime.close()]); + const third = await runtime.close(); + + expect(second).toBe(first); + expect(third).toBe(first); + await expect(runtime.closed).resolves.toBe(first); + expect(first.runners).toEqual({ stopped: ['stopped-device'], handedOff: ['handed-off-device'] }); + expect(first.sessions).toEqual({ closed: 0, failed: 0 }); + expect(first.providerReleases).toEqual({ completed: [], pending: [], unknown: [] }); + expect(first.plugins.disposed).toEqual(['stops']); + expect(first.plugins.failed).toEqual([ + { provider: 'fails', error: expect.objectContaining({ message: 'provider refused to stop' }) }, + ]); + expect(stop).toHaveBeenCalledOnce(); + expect(failing).toHaveBeenCalledOnce(); +}); + +test('a startup that fails after acquiring resources rolls them back once', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-rollback-'); + const stop = vi.fn(async () => {}); + const clearLock = vi.fn(async () => {}); + const failure = new Error('startup step failed'); + const signalListeners = process.listenerCount('SIGTERM'); + + await expect( + startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + providerDeviceRuntimes: [providerRuntime('stops', stop)], + ownerLifecycle: platformOwnerLifecycleFixture({ + configureForDaemonLock: async () => { + throw failure; + }, + clearDaemonLockConfiguration: clearLock, + }), + }), + }), + ).rejects.toBe(failure); + + expect(stop).toHaveBeenCalledOnce(); + expect(clearLock).toHaveBeenCalledOnce(); + expect(process.listenerCount('SIGTERM')).toBe(signalListeners); +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts index 98bf37f55e..aacf50c595 100644 --- a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts @@ -19,7 +19,11 @@ import { interactorResolution, } from '../../core/interactor-resolution.ts'; import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; -import { startDaemonRuntime, type DaemonRuntimeDepsInput } from './daemon-runtime.ts'; +import { + daemonStartupExitCode, + startDaemonRuntime, + type DaemonRuntimeDepsInput, +} from './daemon-runtime.ts'; // The server builds its own store, so capture that instance: the proof is about the store the // server hands to the composition callbacks, not one the test fabricates. @@ -43,7 +47,7 @@ test('daemon startup installs the interactor resolution the composition supplied const stateDir = mkdtempForTestSync('agent-device-daemon-interactor-composition-'); const composed = Object.freeze({ resolve: vi.fn(async () => ({}) as never) }); const previousResolution = interactorResolution(); - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ @@ -54,8 +58,6 @@ test('daemon startup installs the interactor resolution the composition supplied AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ interactorResolution: composed }), @@ -66,7 +68,7 @@ test('daemon startup installs the interactor resolution the composition supplied await interactorResolution().resolve(IOS_SIMULATOR, {}); expect(composed.resolve).toHaveBeenCalledOnce(); } finally { - await runtime?.shutdown(); + await runtime?.close(); installInteractorResolution(previousResolution); fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -80,11 +82,8 @@ test('a daemon attempt losing the lock shuts down every constructed provider', a throw new Error('cleanup failed'); }); const otherShutdown = vi.fn(async () => {}); - const exit = vi.fn(); - const runtime = await startDaemonRuntime({ + const refusal = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { AGENT_DEVICE_STATE_DIR: stateDir } }), - exit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ @@ -97,11 +96,10 @@ test('a daemon attempt losing the lock shuts down every constructed provider', a }) as unknown as ProviderDeviceRuntime, ), }), - }); - expect(runtime).toBeNull(); + }).catch((error: unknown) => error); + expect(daemonStartupExitCode(refusal)).toBe(DAEMON_STARTUP_EXIT_CODES.busy); expect(shutdown).toHaveBeenCalledOnce(); expect(otherShutdown).toHaveBeenCalledOnce(); - expect(exit).toHaveBeenCalledWith(DAEMON_STARTUP_EXIT_CODES.busy); await held.owner.finish(); }); @@ -119,8 +117,6 @@ test('the server runs on the host it was given and hands that host to the compos const fixtureDeps = daemonRuntimeDepsFixture(); const runtime = await startDaemonRuntime({ host, - registerProcessHandlers: false, - exit: () => {}, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: async (input) => { @@ -132,7 +128,7 @@ test('the server runs on the host it was given and hands that host to the compos assert.equal(composed?.host, host); assert.equal(composed?.baseDir, resolveDaemonPaths(stateDir).baseDir); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -150,10 +146,9 @@ test('the server derives the policy gate and session callbacks and hands them to JSON.stringify({ version: 1, capabilities: { deny: ['device-shutdown'] } }), ); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; let composed: DaemonRuntimeDepsInput | undefined; const fixtureDeps = daemonRuntimeDepsFixture(); - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ @@ -164,8 +159,6 @@ test('the server derives the policy gate and session callbacks and hands them to AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: async (input) => { @@ -173,8 +166,6 @@ test('the server derives the policy gate and session callbacks and hands them to return await fixtureDeps(input); }, }); - assert.notEqual(runtime, null, 'the assembly must boot'); - assert.deepEqual(exits, []); assert.ok(composed, 'the server must call the factory'); // The server resolved the paths and loaded the operator's policy before composing anything. assert.equal(composed.baseDir, paths.baseDir); @@ -206,11 +197,10 @@ test('the server derives the policy gate and session callbacks and hands them to outputPath: store.resolveAppLogPath('web-assembly'), pidPath: store.resolveAppLogPidPath('web-assembly'), }); - await runtime?.shutdown(); + await runtime?.close(); runtime = null; - assert.deepEqual(exits, [0]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -220,12 +210,9 @@ test('the server derives the policy gate and session callbacks and hands them to */ test('an unreadable policy stops the assembly before the factory is called', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-assembly-bad-policy-'); - const exits: number[] = []; - const stderr: string[] = []; let factoryCalls = 0; - let runtime: Awaited> = null; try { - runtime = await startDaemonRuntime({ + const refusal = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { ...process.env, @@ -234,20 +221,17 @@ test('an unreadable policy stops the assembly before the factory is called', asy AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - stderr: { write: (chunk) => stderr.push(chunk) }, + stderr: { write: () => {} }, stdout: { write: () => {} }, - registerProcessHandlers: false, composeDeps: async (input) => { factoryCalls += 1; return await daemonRuntimeDepsFixture()(input); }, - }); - assert.equal(runtime, null); - assert.deepEqual(exits, [1]); + }).catch((error: unknown) => error); + assert.equal(daemonStartupExitCode(refusal), 1); assert.equal(factoryCalls, 0, 'no runtime may be built under an unreadable policy'); - assert.match(stderr.join(''), /Daemon error: .*/); + assert.match((refusal as Error).message, /policy/i); } finally { - await runtime?.shutdown(); + fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts b/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts index af6061195a..f27d9d84f7 100644 --- a/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts @@ -30,11 +30,7 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-rt-'); const paths = resolveDaemonPaths(stateDir); - let exitCode: number | undefined; - let resolveExit: () => void; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -47,25 +43,22 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '80', }, }), - exit: (code) => { - exitCode = code; - resolveExit(); - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); assert.ok(fs.existsSync(paths.lockPath), 'daemon lock should be held right after startup'); - assert.equal(exitCode, undefined); + assert.equal(closed, false); await vi.advanceTimersByTimeAsync(79); - assert.equal(exitCode, undefined); + assert.equal(closed, false); await vi.advanceTimersByTimeAsync(1); await vi.runOnlyPendingTimersAsync(); await exited; - assert.equal(exitCode, 0); + assert.equal(closed, true); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -76,7 +69,7 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS is 0', async () => { vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-off-'); - let exitCode: number | undefined; + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -89,21 +82,20 @@ test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS i AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); await vi.advanceTimersByTimeAsync(200); - assert.equal(exitCode, undefined); - const shutdownPromise = runtime?.shutdown(); + assert.equal(closed, false); + const shutdownPromise = runtime.close(); await vi.runOnlyPendingTimersAsync(); await shutdownPromise; - assert.equal(exitCode, 0); + await exited; + assert.equal(closed, true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -112,11 +104,7 @@ test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS i test('only a retained lease keeps the daemon runtime from self-reaping', async () => { vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-leases-'); - let exitCode: number | undefined; - let resolveExit: () => void; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -129,15 +117,12 @@ test('only a retained lease keeps the daemon runtime from self-reaping', async ( AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '80', }, }), - exit: (code) => { - exitCode = code; - resolveExit(); - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); const [registry] = leaseProbe.registries; registry!.allocateLease({ tenantId: 'tenant-a', runId: 'run-plain', ttlMs: 60_000 }); const retained = registry!.allocateLease({ @@ -148,13 +133,13 @@ test('only a retained lease keeps the daemon runtime from self-reaping', async ( }); await vi.advanceTimersByTimeAsync(80); - assert.equal(exitCode, undefined); + assert.equal(closed, false); registry!.releaseLease({ leaseId: retained.leaseId }); await vi.advanceTimersByTimeAsync(80); await vi.runOnlyPendingTimersAsync(); await exited; - assert.equal(exitCode, 0); + assert.equal(closed, true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts index 3d2892f5a8..e059b1d2f8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts @@ -63,8 +63,6 @@ test('an expired lease ends its session without any further request', async () = AGENT_DEVICE_LEASE_MIN_TTL_MS: '1', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -78,7 +76,7 @@ test('an expired lease ends its session without any further request', async () = await vi.advanceTimersByTimeAsync(1_000); assert.deepEqual(leaseProbe.ended, [[lease.leaseId, 'LEASE_EXPIRED']]); - const shutdown = runtime?.shutdown(); + const shutdown = runtime?.close(); await vi.runOnlyPendingTimersAsync(); await shutdown; } finally { @@ -106,14 +104,12 @@ test('shutdown cancels and drains lease ends before it tears sessions down', asy AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); assert.notEqual(runtime, null); - await runtime?.shutdown(); + await runtime?.close(); assert.deepEqual(leaseProbe.shutdownSteps, ['cancel', 'drain', 'teardown']); } finally { diff --git a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts index 7b506c5324..5449f86328 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts @@ -116,7 +116,6 @@ afterEach(() => { test('daemon shutdown detaches before session teardown and force-finalizes only after gateway resources', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-shutdown-'); - const exits: number[] = []; const startupErrors: string[] = []; try { const runtime = await startDaemonRuntime({ @@ -135,9 +134,11 @@ test('daemon shutdown detaches before session teardown and force-finalizes only phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -153,17 +154,14 @@ test('daemon shutdown detaches before session teardown and force-finalizes only AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - registerProcessHandlers: false, stderr: { write: (chunk) => startupErrors.push(chunk) }, stdout: { write: () => {} }, }); expect(runtime, startupErrors.join('')).not.toBeNull(); - await Promise.all([runtime?.shutdown(), runtime?.shutdown()]); + await Promise.all([runtime?.close(), runtime?.close()]); expect(lifecycleEvents).toEqual(['detach', 'gateway-shutdown', 'finalize']); - expect(exits).toEqual([0]); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -190,9 +188,11 @@ test('a SIGTERM shutdown gives the handoff a diagnostics scope to write its reas phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -208,14 +208,12 @@ test('a SIGTERM shutdown gives the handoff a diagnostics scope to write its reas AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); expect(runtime).not.toBeNull(); - await runtime?.shutdown(); + await runtime?.close(); const daemonLog = fs.readFileSync(path.join(stateDir, 'daemon.log'), 'utf8'); expect(daemonLog).toMatch(/"phase":"detach_scope_probe"/); @@ -245,9 +243,11 @@ test('shutdown includes drain publications, releases their claims and refuses po phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -263,8 +263,6 @@ test('shutdown includes drain publications, releases their claims and refuses po AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -298,14 +296,14 @@ test('shutdown includes drain publications, releases their claims and refuses po refusal = error; } }); - await runtime!.shutdown(); + await runtime!.close(); expect(shutdownProbe.finalize).toHaveBeenCalledOnce(); expect(refusal).toMatchObject({ details: { reason: 'daemon_shutting_down' } }); expect(store.lookup('draining')).toBeUndefined(); expect(store.lookup('late')).toBeUndefined(); expect(fs.existsSync(deviceClaimFilePath(acquired.ownership.deviceKey))).toBe(false); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -327,9 +325,11 @@ test('shutdown joins dispatch completion after force-closing the client before t phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -345,8 +345,6 @@ test('shutdown joins dispatch completion after force-closing the client before t AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -363,7 +361,7 @@ test('shutdown joins dispatch completion after force-closing the client before t closed = resolve; }); let request: Promise | undefined; - let shutdown: Promise | undefined; + let shutdown: Promise | undefined; try { expect(runtime).not.toBeNull(); const store = shutdownProbe.store!; @@ -413,7 +411,7 @@ test('shutdown joins dispatch completion after force-closing the client before t shutdownProbe.drain = async () => { closed(); }; - shutdown = runtime!.shutdown(); + shutdown = runtime!.close(); await serversClosed; await new Promise((resolve) => setImmediate(resolve)); expect(lifecycleEvents).not.toContain('detach'); @@ -429,7 +427,7 @@ test('shutdown joins dispatch completion after force-closing the client before t release(); await request; await shutdown; - await runtime?.shutdown(); + await runtime?.close(); } }); test.each([false, true])( @@ -496,9 +494,11 @@ test('daemon shutdown releases a session lease whose session teardown rejects', phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -514,8 +514,6 @@ test('daemon shutdown releases a session lease whose session teardown rejects', AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -531,7 +529,7 @@ test('daemon shutdown releases a session lease whose session teardown rejects', }); shutdownProbe.finalize.mockRejectedValueOnce(new Error('teardown failed')); - await runtime?.shutdown(); + await runtime?.close(); expect(shutdownProbe.finalize).toHaveBeenCalledOnce(); expect(leaseProbe.released).toEqual([lease]); @@ -557,9 +555,11 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -575,8 +575,6 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -589,7 +587,7 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy retainOnClose: true, }); - await runtime?.shutdown(); + await runtime?.close(); expect(leaseProbe.released).toEqual([lease]); expect(leaseRegistry!.listActiveLeases()).toEqual([]); diff --git a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts index a873610674..d1c1592d47 100644 --- a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts @@ -28,7 +28,7 @@ type DaemonPaths = ReturnType; // anything that verifies identity against the process table. const SUCCESSOR_PID = 999_999_999; -function startRuntime(stateDir: string, exit: (code: number) => void) { +function startRuntime(stateDir: string) { return startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(() => ({ // The post-lock, pre-publication step the runtime awaits first. Making it throw lands the @@ -56,8 +56,6 @@ function startRuntime(stateDir: string, exit: (code: number) => void) { AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -90,15 +88,13 @@ test('a shutdown whose daemon.json names a successor keeps the file and logs the // daemon's metadata and leaving every later client failing with hasInfo:false + retainedLockProcess. const stateDir = mkdtempForTestSync('agent-device-daemon-info-ownership-shutdown-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); publishSuccessor(paths); - await runtime?.shutdown(); + await runtime?.close(); - expect(exits).toEqual([0]); expect(lifecycleEvents).toContain('gateway-shutdown'); expect(fs.existsSync(paths.infoPath), 'the serving daemon keeps its metadata').toBe(true); expect(fs.existsSync(paths.lockPath), 'the exiting daemon still releases its own lock').toBe( @@ -119,7 +115,7 @@ test('a shutdown removes its own metadata and reports an unverified release', as const stateDir = mkdtempForTestSync('agent-device-daemon-info-owned-shutdown-'); const paths = resolveDaemonPaths(stateDir); try { - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); const originalRmdir = fs.rmdirSync; @@ -127,7 +123,7 @@ test('a shutdown removes its own metadata and reports an unverified release', as if (target === paths.lockPath) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); return originalRmdir(target, options); }); - await runtime?.shutdown(); + await runtime?.close(); expect(fs.existsSync(paths.infoPath)).toBe(false); expect(logEvents(stateDir).map((event) => event.phase)).not.toContain( @@ -159,12 +155,12 @@ test('a skipped provider runtime is logged after this daemon publishes its recor code: 'INVALID_ARGS', }); providerComposition.skipped = [{ provider: 'limrun', error }]; - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); try { const skipped = logEvents(stateDir).find(({ phase }) => phase === 'provider_runtime_skipped'); expect(skipped?.data).toMatchObject({ provider: 'limrun', code: 'INVALID_ARGS' }); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -173,13 +169,9 @@ test('a startup that fails before publication removes no record and writes no lo // silent: reporting an "absent" record here would create a daemon.log on every such startup. const stateDir = mkdtempForTestSync('agent-device-daemon-info-startup-absent-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; startupFailure.active = true; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); - - expect(runtime).toBeNull(); - expect(exits).toEqual([1]); + await expect(startRuntime(stateDir)).rejects.toThrow('startup step failed'); expect(fs.existsSync(paths.infoPath)).toBe(false); expect(fs.existsSync(paths.logPath), 'nothing was declined, so nothing is logged').toBe(false); } finally { @@ -190,14 +182,10 @@ test('a startup that fails before publication removes no record and writes no lo test('a startup that fails before publication leaves a foreign record standing', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-info-startup-foreign-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; publishSuccessor(paths); startupFailure.active = true; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); - - expect(runtime).toBeNull(); - expect(exits).toEqual([1]); + await expect(startRuntime(stateDir)).rejects.toThrow('startup step failed'); expect(fs.existsSync(paths.infoPath)).toBe(true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); @@ -213,7 +201,7 @@ test('a successor published during this daemon lifetime is recorded at detection const stateDir = mkdtempForTestSync('agent-device-daemon-metadata-lost-'); const paths = resolveDaemonPaths(stateDir); try { - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); publishSuccessor(paths); @@ -227,7 +215,7 @@ test('a successor published during this daemon lifetime is recorded at detection ); vi.useRealTimers(); - await runtime?.shutdown(); + await runtime?.close(); } finally { vi.useRealTimers(); fs.rmSync(stateDir, { recursive: true, force: true }); @@ -249,13 +237,14 @@ test('both exits tear the watch down before they touch daemon.json', () => { // A watch still armed when the removal runs can report a loss for a record its own process just // deleted. The startup-failure half of this is unreachable from a test — everything after // publication needs a real toolchain — so the invariant is read off the source, the same way the - // arming order above is. + // arming order above is. Startup rollback runs its steps in reverse, so the registration that + // stops the watch comes after the one that finishes the registration; the close path stops first. const source = fs.readFileSync(new URL('./daemon-runtime.ts', import.meta.url), 'utf8'); const stops = [...source.matchAll(/stopMetadataLossWatch\(\);/g)].map((m) => m.index); const finishes = [...source.matchAll(/await finishDaemonRegistration\(/g)].map((m) => m.index); expect(stops).toHaveLength(2); expect(finishes).toHaveLength(2); - expect(finishes[0]).toBeGreaterThan(stops[0]!); - expect(finishes[0]).toBeLessThan(stops[1]!); + expect(stops[0]).toBeGreaterThan(finishes[0]!); + expect(stops[1]).toBeGreaterThan(stops[0]!); expect(finishes[1]).toBeGreaterThan(stops[1]!); }); diff --git a/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts b/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts index a838601426..21e1c4f71e 100644 --- a/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts @@ -33,8 +33,6 @@ function startDaemon( AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ @@ -70,7 +68,7 @@ test('daemon startup configures its platform owners before it publishes readines }), ]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -94,6 +92,6 @@ test('a platform owner diagnostic about a refused restore is logged and the daem }), ]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts b/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts index 8304b4c083..ed04ed819e 100644 --- a/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts @@ -35,8 +35,6 @@ test('daemon startup reaps orphaned simctl recorders with a graceful finalize wi AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -50,7 +48,7 @@ test('daemon startup reaps orphaned simctl recorders with a graceful finalize wi // so it releases the host recording lock instead of leaving it dangling for the next session. expect(Number(recordingReap!.termTimeoutMs)).toBeGreaterThanOrEqual(5_000); - await runtime?.shutdown(); + await runtime?.close(); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts b/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts index ec814ad7e9..1526f796b8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts @@ -52,7 +52,6 @@ test('a /rpc flow failure the server refused itself reaches the runtime stderr a expect(observed.events).toHaveLength(1); expect(observed.events[0]?.data).toEqual({ code: 'UNKNOWN', message: 'Unknown error' }); expect(observed.rejections).toEqual([]); - expect(observed.exitCode).toBeUndefined(); expect(observed.nextCall).toEqual({ status: 400, code: -32600 }); }); @@ -67,7 +66,6 @@ test('a /rpc flow failure still reaches daemon.log when the injected stderr thro expect(observed.events[0]?.phase).toBe('daemon_rpc_request_flow_failed'); expect(observed.stderrLines).toEqual([]); expect(observed.rejections).toEqual([]); - expect(observed.exitCode).toBeUndefined(); }); type FlowFailure = { @@ -76,7 +74,6 @@ type FlowFailure = { events: Array<{ phase: string; data?: Record }>; stderrLines: string[]; rejections: unknown[]; - exitCode: number | undefined; }; /** @@ -94,8 +91,7 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise { rejections.push(reason); }; - let exitCode: number | undefined; - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(), @@ -107,10 +103,6 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: (chunk: string) => { if (options.stderrThrows) throw new Error('the stream is gone'); @@ -143,14 +135,13 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise ({ error }), ); const stderr: string[] = []; - let teardown: Promise | undefined; + let teardown: Promise | undefined; try { await captureEntered.promise; vi.useFakeTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 9e245c567c..143f8889da 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -39,7 +39,13 @@ import { stopSessionAppLog, teardownSessionResources } from '../../sessions/sess import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; import { finalizeDaemonSessionApplicationLifecycle } from '../application-lifecycle-recovery.ts'; import { runtimeHintValues } from '../../sessions/session-runtime.ts'; -import { closeDaemonServers } from './server-shutdown.ts'; +import { + closeDaemonServers, + createProviderRuntimeDisposal, + createStartupRollback, + type ShutdownReport, + type StartupRollback, +} from './server-shutdown.ts'; import type { DaemonInvokeFn } from '../../core/daemon-request.ts'; import type { SessionRef, SessionState } from '../../core/session-state.ts'; import type { RuntimeHintValues } from '@agent-device/contracts/application-lifecycle-runtime'; @@ -198,8 +204,6 @@ export type DaemonRuntimeOptions = { host: HostEnvironment; stdout?: WritableOutput; stderr?: WritableOutput; - exit?: (code: number) => void; - registerProcessHandlers?: boolean; /** The root's runtime composition, as a factory so the server owns paths, policy, and store. */ composeDeps: ComposeDaemonRuntimeDeps; }; @@ -207,10 +211,25 @@ export type DaemonRuntimeOptions = { export type DaemonRuntimeController = { httpPort?: number; socketPort?: number; - shutdown: (options?: { exitCode?: number; cause?: unknown }) => Promise; + /** + * Closes the runtime. Concurrent and repeated calls share one close, whose report every caller + * receives. The runtime never exits the process: the executable owns signals and exit codes. + */ + close: (options?: { cause?: unknown }) => Promise; + /** Settles once the runtime has closed, whoever closed it (an idle reap closes it unasked). */ + closed: Promise; token: string; }; +/** + * Thrown by {@link startDaemonRuntime} when the daemon must not run: its policy is unusable or + * another daemon owns the state dir. `details.exitCode` is the code the executable exits with. + */ +export function daemonStartupExitCode(error: unknown): number { + const exitCode = asAppError(error).details?.exitCode; + return typeof exitCode === 'number' ? exitCode : 1; +} + export async function flushDaemonStartupDiagnostics( logPath: string, diagnostics: readonly ResourceDiagnostic[], @@ -326,7 +345,7 @@ export async function teardownDaemonSessionForShutdown(params: { runtimeHints: RuntimeHintValues, ) => Promise; afterSuccessfulTeardown?: (session: SessionState) => Promise; -}): Promise { +}): Promise { const { ref, sessionStore, @@ -390,24 +409,31 @@ export async function teardownDaemonSessionForShutdown(params: { sessionStore.finalizeRepairTeardown(ref); if (teardownSucceeded) await afterSuccessfulTeardown?.(session); sessionStore.retire(ref); + return teardownSucceeded; } export async function startDaemonRuntime( options: DaemonRuntimeOptions, -): Promise { +): Promise { return await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { - return await startDaemonRuntimeOnHost(options); + const rollback = createStartupRollback(); + try { + return await startDaemonRuntimeOnHost(options, rollback); + } catch (error) { + await rollback.run(); + throw error; + } }); } async function startDaemonRuntimeOnHost( options: DaemonRuntimeOptions, -): Promise { + rollback: StartupRollback, +): Promise { const { host } = options; const { env } = host; const stdout = options.stdout ?? process.stdout; const stderr = options.stderr ?? process.stderr; - const exit = options.exit ?? ((code: number) => process.exit(code)); const daemonPaths = resolveDaemonPaths(env.AGENT_DEVICE_STATE_DIR, { env, cwd: host.cwd }); const { baseDir, infoPath, logPath, sessionsDir } = daemonPaths; const daemonServerMode = resolveDaemonServerMode(env.AGENT_DEVICE_DAEMON_SERVER_MODE); @@ -415,14 +441,7 @@ async function startDaemonRuntimeOnHost( // ADR 0029: a policy that cannot be read or validated stops startup; the daemon never runs // with a weaker policy than its operator named. The server loads it before composing any // runtime, so no capability is ever built under a policy that failed validation. - let daemonPolicy: DaemonPolicy | undefined; - try { - daemonPolicy = loadDaemonPolicy(env); - } catch (error) { - stderr.write(`Daemon error: ${asAppError(error).message}\n`); - exit(1); - return null; - } + const daemonPolicy: DaemonPolicy | undefined = loadDaemonPolicy(env); // Derived here, beside the policy it reads: the server hands the same refusal to the root's // composition, so a runtime can never be built without the operator's shutdown gate. const assertShutdownAllowed = daemonPolicy @@ -452,6 +471,10 @@ async function startDaemonRuntimeOnHost( const daemonCodeOrigin = resolveDaemonCodeOrigin(); const daemonCodeSignature = resolveDaemonCodeSignature(); const providerDeviceRuntimes = deps.providerDeviceRuntimes; + const pluginDisposal = createProviderRuntimeDisposal(providerDeviceRuntimes); + rollback.onRollback(async () => { + await pluginDisposal.dispose(); + }); const deviceRuntimeGateway = deps.deviceRuntimeGateway; const applicationLifecycle = deviceRuntimeGateway.applicationLifecycle; if (!applicationLifecycle) { @@ -468,6 +491,7 @@ async function startDaemonRuntimeOnHost( stateDir: baseDir, recoverableProviderIds: deps.recoverableProviderIds, }); + rollback.onRollback(() => expiredProviderLeaseReleaser.shutdown()); void expiredProviderLeaseReleaser.retryPending(); const leaseRegistry = new LeaseRegistry({ maxActiveSimulatorLeases: parseIntegerEnv(env.AGENT_DEVICE_MAX_SIMULATOR_LEASES), @@ -486,6 +510,7 @@ async function startDaemonRuntimeOnHost( LEASE_EXPIRY_SWEEP_INTERVAL_MS, ); leaseExpirySweep.unref(); + rollback.onRollback(() => clearInterval(leaseExpirySweep)); const dispatchRequest = createRequestHandler({ host, logPath, @@ -525,10 +550,10 @@ async function startDaemonRuntimeOnHost( const shutdownClaimLedger = createDaemonShutdownClaimLedger(); - const teardownDaemonSession = async (ref: SessionRef): Promise => { + const teardownDaemonSession = async (ref: SessionRef): Promise => { const session = sessionStore.resolveCurrent(ref) ?? ref.session; try { - await teardownDaemonSessionForShutdown({ + return await teardownDaemonSessionForShutdown({ ref, sessionStore, stderr, @@ -548,10 +573,14 @@ async function startDaemonRuntimeOnHost( } }; - const teardownDaemonSessions = async (): Promise => { + const teardownDaemonSessions = async (): Promise => { sessionStore.closeAdmission(); const sessionsToStop = sessionStore.listRefs(); - await Promise.all(sessionsToStop.map(teardownDaemonSession)); + const outcomes = await Promise.all( + sessionsToStop.map(async (ref) => await teardownDaemonSession(ref).catch(() => false)), + ); + const closedCount = outcomes.filter(Boolean).length; + return { closed: closedCount, failed: outcomes.length - closedCount }; }; // #2833: settles the resources of a session this daemon expires for idleness. Deliberately NOT @@ -621,7 +650,7 @@ async function startDaemonRuntimeOnHost( // Reaps this daemon process when it sits fully idle (no open sessions, no // in-flight requests, no active recording) past AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS. - // `shutdown` is defined below but only invoked asynchronously by the timer, + // `close` is defined below but only invoked asynchronously by the timer, // well after this closure captures it. let shuttingDown = false; const inFlightRequests = new Set>(); @@ -630,11 +659,17 @@ async function startDaemonRuntimeOnHost( getInFlightRequestCount: () => inFlightRequests.size, hasRetainedLeases: () => leaseRegistry.hasRetainedLeases(), onIdleReap: () => { - void shutdown(); + void close(); }, env, }); + rollback.onRollback(() => { + idleReap.cancel(); + sessionIdleExpiry.cancel(); + leaseSessionEnd.cancel(); + }); + const handleRequest: DaemonInvokeFn = async (req) => { if (shuttingDown) return { @@ -741,129 +776,119 @@ async function startDaemonRuntimeOnHost( const acquisition = await tryAcquireDaemonRegistration(daemonPaths); if (acquisition.status !== 'acquired') { - await Promise.allSettled( - providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), - ); - stderr.write(`Daemon registration ${acquisition.status}; exiting.\n`); - exit(DAEMON_STARTUP_EXIT_CODES[acquisition.status]); - return null; + throw new AppError('COMMAND_FAILED', `Daemon registration ${acquisition.status}.`, { + reason: 'daemon_registration_refused', + status: acquisition.status, + exitCode: DAEMON_STARTUP_EXIT_CODES[acquisition.status], + }); } const registration = acquisition.owner; - + rollback.onRollback(async () => { + await deps.ownerLifecycle.clearDaemonLockConfiguration(); + }); + rollback.onRollback(async () => { + await finishDaemonRegistration({ registration, infoPath, logPath }); + }); let servers: DaemonServer[] = []; - let socketPort: number | undefined; - let httpPort: number | undefined; + rollback.onRollback(async () => { + closeServersBestEffort(servers); + stopMetadataLossWatch(); + }); const startupDiagnostics: ResourceDiagnostic[] = []; - try { - await deps.ownerLifecycle.configureForDaemonLock({ - stateDir: baseDir, - hasDeviceClaimAuthority: (device) => processOwnsActiveDeviceClaim(device, env), - onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), - }); - const legacyMarkerRecovery = await deps.ownerLifecycle.recoverLegacyAppLogMarkers(sessionsDir); - appLogAdmissionLedger.retainLegacyMarkers(legacyMarkerRecovery.retained); - for (const markerPath of legacyMarkerRecovery.recovered) { - startupDiagnostics.push({ - phase: 'app_log_legacy_marker_recovered', - resourcePath: markerPath, - data: {}, - }); - } - for (const retained of legacyMarkerRecovery.retained) { - startupDiagnostics.push({ - phase: 'app_log_legacy_marker_retained', - resourcePath: retained.markerPath, - data: { - reason: retained.reason, - ...(retained.message === undefined ? {} : { message: retained.message }), - }, - }); - } - await recoverAppLogResourcesAfterDaemonLock({ - sessionsDir, - gateway: deviceRuntimeGateway, - scope: createRecoveryPlatformScope(), - onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + await deps.ownerLifecycle.configureForDaemonLock({ + stateDir: baseDir, + hasDeviceClaimAuthority: (device) => processOwnsActiveDeviceClaim(device, env), + onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + }); + const legacyMarkerRecovery = await deps.ownerLifecycle.recoverLegacyAppLogMarkers(sessionsDir); + appLogAdmissionLedger.retainLegacyMarkers(legacyMarkerRecovery.retained); + for (const markerPath of legacyMarkerRecovery.recovered) { + startupDiagnostics.push({ + phase: 'app_log_legacy_marker_recovered', + resourcePath: markerPath, + data: {}, }); - await reapOwnedProcessRecordsAtStartup(deps.ownedProcessRecords, { - openWebSessionNames: openWebSessionNames(sessionStore), - purposes: ['simctl-screen-recording'], - termTimeoutMs: DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS, + } + for (const retained of legacyMarkerRecovery.retained) { + startupDiagnostics.push({ + phase: 'app_log_legacy_marker_retained', + resourcePath: retained.markerPath, + data: { + reason: retained.reason, + ...(retained.message === undefined ? {} : { message: retained.message }), + }, }); - await deps.cleanupManagedWebOrphans({ - stateDir: baseDir, - openWebSessionNames: openWebSessionNames(sessionStore), - ownedProcessRecords: deps.ownedProcessRecords, + } + await recoverAppLogResourcesAfterDaemonLock({ + sessionsDir, + gateway: deviceRuntimeGateway, + scope: createRecoveryPlatformScope(), + onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + }); + await reapOwnedProcessRecordsAtStartup(deps.ownedProcessRecords, { + openWebSessionNames: openWebSessionNames(sessionStore), + purposes: ['simctl-screen-recording'], + termTimeoutMs: DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS, + }); + await deps.cleanupManagedWebOrphans({ + stateDir: baseDir, + openWebSessionNames: openWebSessionNames(sessionStore), + ownedProcessRecords: deps.ownedProcessRecords, + }); + // Marker-gated lifecycle recovery owns test-IME orphan repair. Its implementation remains + // lazy until the marker exists, so a normal daemon startup does not load or probe adb. + void applicationLifecycle.recoverStartupResources({ stateDir: baseDir }).catch((error) => { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_lifecycle_startup_recovery_failed', + data: { error: error instanceof Error ? error.message : String(error) }, }); - // Marker-gated lifecycle recovery owns test-IME orphan repair. Its implementation remains - // lazy until the marker exists, so a normal daemon startup does not load or probe adb. - void applicationLifecycle.recoverStartupResources({ stateDir: baseDir }).catch((error) => { - emitDiagnostic({ - level: 'warn', - phase: 'daemon_lifecycle_startup_recovery_failed', - data: { error: error instanceof Error ? error.message : String(error) }, - }); + }); + const opened = await openDaemonServers(); + servers = opened.servers; + const { socketPort, httpPort } = opened; + publishDaemonInfo(socketPort, httpPort); + stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); + await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); + for (const skipped of deps.skippedProviderRuntimes) { + await emitDaemonDiagnostic(logPath, 'provider_runtime_skipped', { + provider: skipped.provider, + code: skipped.error.code, + message: skipped.error.message, + hint: skipped.error.details?.hint, }); - const opened = await openDaemonServers(); - servers = opened.servers; - socketPort = opened.socketPort; - httpPort = opened.httpPort; - publishDaemonInfo(socketPort, httpPort); - stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); - await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); - for (const skipped of deps.skippedProviderRuntimes) { - await emitDaemonDiagnostic(logPath, 'provider_runtime_skipped', { - provider: skipped.provider, - code: skipped.error.code, - message: skipped.error.message, - hint: skipped.error.details?.hint, - }); - } - // After publication: publishDaemonInfo truncates daemon.log, so anything - // written before it is lost — including reconciliation diagnostics. - await reconcileDeviceClaimsForDaemonStartup( - logPath, - createOwnerScopedDeviceClaimReconciler({ - scope: createRecoveryPlatformScope(), - composeGateway: deps.claimRecoveryGateway, - }), - baseDir, - env, - ); - // Arms the initial idle-reap timer: a daemon that starts and never - // receives a request must still be able to reap itself. - idleReap.noteActivity(); - // The #2833 deadline needs no equivalent arm: the store is empty until a request puts a session - // in it, and that request's own completion re-arms the reaper. - } catch (error) { - const appErr = asAppError(error); - stderr.write(`Daemon error: ${appErr.message}\n`); - closeServersBestEffort(servers); - stopMetadataLossWatch(); - await Promise.allSettled( - providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), - ); - await finishDaemonRegistration({ registration, infoPath, logPath }); - await deps.ownerLifecycle.clearDaemonLockConfiguration(); - exit(1); - return null; } + // After publication: publishDaemonInfo truncates daemon.log, so anything + // written before it is lost — including reconciliation diagnostics. + await reconcileDeviceClaimsForDaemonStartup( + logPath, + createOwnerScopedDeviceClaimReconciler({ + scope: createRecoveryPlatformScope(), + composeGateway: deps.claimRecoveryGateway, + }), + baseDir, + env, + ); + // Arms the initial idle-reap timer: a daemon that starts and never + // receives a request must still be able to reap itself. + idleReap.noteActivity(); + // The #2833 deadline needs no equivalent arm: the store is empty until a request puts a session + // in it, and that request's own completion re-arms the reaper. // Spawn the PNG worker ahead of the first screenshot request so its // cold-start cost is not paid on a user-visible call. Best effort: when it // cannot start, PNG processing falls back to the in-process sync path. prewarmPngWorker(); - const shutdownOnHost = async (shutdownOptions: { exitCode?: number; cause?: unknown } = {}) => { + const closeOnHost = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); sessionIdleExpiry.cancel(); leaseSessionEnd.cancel(); clearInterval(leaseExpirySweep); - if (shuttingDown) return; shuttingDown = true; stopMetadataLossWatch(); - if (shutdownOptions.cause) { - await emitFatalDiagnostic(shutdownOptions.cause); + if (closeOptions.cause) { + await emitFatalDiagnostic(closeOptions.cause); } await closeDaemonServers(servers); await Promise.allSettled(inFlightRequests); @@ -875,13 +900,14 @@ async function startDaemonRuntimeOnHost( // no diagnostics scope, so `emitDiagnostic` would drop every detach reason. Daemon debug level is // forced on here because the declines are the point of the record — a handoff that silently // skipped is indistinguishable from a rebuild (#2681). + let handedOffRunners: readonly string[] = []; try { await withDaemonDiagnosticsScope({ logPath }, async () => { - await applicationLifecycle.detachForDaemonShutdown(); + handedOffRunners = await applicationLifecycle.detachForDaemonShutdown(); }); } catch {} expiredProviderLeaseReleaser.beginShutdown(); - await teardownDaemonSessions(); + const sessions = await teardownDaemonSessions(); await withDaemonDiagnosticsScope({ logPath }, async () => { await finalizeDaemonLeases({ leaseRegistry, @@ -913,10 +939,8 @@ async function startDaemonRuntimeOnHost( }); expiredProviderLeaseReleaser.shutdown(); await deviceRuntimeGateway.shutdown(); - await Promise.allSettled( - providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), - ); - await applicationLifecycle.finalizeDaemonShutdown(); + const plugins = await pluginDisposal.dispose(); + const stoppedRunners = await applicationLifecycle.finalizeDaemonShutdown(); // Best effort: stop the PNG worker so an in-flight job cannot delay exit. await Promise.race([ terminatePngWorker().catch(() => {}), @@ -929,42 +953,46 @@ async function startDaemonRuntimeOnHost( outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, }); await deps.ownerLifecycle.clearDaemonLockConfiguration(); - exit(shutdownOptions.exitCode ?? 0); + const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); + return { + providerReleases: { + completed: providerReleaseDrain.released.map((lease) => lease.leaseId), + pending: providerReleaseDrain.pending + .filter((lease) => !unknownLeaseIds.has(lease.leaseId)) + .map((lease) => lease.leaseId), + unknown: [...unknownLeaseIds], + }, + runners: { stopped: [...stoppedRunners], handedOff: [...handedOffRunners] }, + sessions, + plugins, + }; }; - // Signal handlers and external callers run outside the startup scope, so shutdown re-enters it. - const shutdown = async (shutdownOptions: { exitCode?: number; cause?: unknown } = {}) => - await withCommandHost( - { host, requestEnv: undefined }, - async () => await shutdownOnHost(shutdownOptions), - ); - - if (options.registerProcessHandlers !== false) { - process.on('SIGINT', () => { - void shutdown(); - }); - process.on('SIGTERM', () => { - void shutdown(); - }); - process.on('SIGHUP', () => { - void shutdown(); - }); - process.on('uncaughtException', (err) => { - const appErr = err instanceof AppError ? err : asAppError(err); - stderr.write(`Daemon error: ${appErr.message}\n`); - void shutdown({ exitCode: 1, cause: err }); - }); - process.on('unhandledRejection', (reason) => { - const err = reason instanceof Error ? reason : new Error(String(reason)); - const appErr = err instanceof AppError ? err : asAppError(err); - stderr.write(`Daemon error: ${appErr.message}\n`); - void shutdown({ exitCode: 1, cause: err }); - }); - } + // Idle reap, external callers and the executable's signal handlers all close through here, and + // none of them runs inside the startup scope, so the close re-enters it. + let closing: Promise | undefined; + let settleClosed: { resolve(report: ShutdownReport): void; reject(error: unknown): void }; + const closed = new Promise((resolve, reject) => { + settleClosed = { resolve, reject }; + }); + // A close nobody awaits (an idle reap) still reports through `closed`, which must not become an + // unhandled rejection when no caller observes it. + closed.catch(() => {}); + const close = (closeOptions: { cause?: unknown } = {}): Promise => { + if (!closing) { + closing = withCommandHost( + { host, requestEnv: undefined }, + async () => await closeOnHost(closeOptions), + ); + closing.then(settleClosed.resolve, settleClosed.reject); + } + return closing; + }; return { httpPort, - shutdown, + close, + closed, socketPort, token, }; diff --git a/packages/daemon/src/server/http/server-shutdown.ts b/packages/daemon/src/server/http/server-shutdown.ts index b86f71a670..fe7a81de2d 100644 --- a/packages/daemon/src/server/http/server-shutdown.ts +++ b/packages/daemon/src/server/http/server-shutdown.ts @@ -1,3 +1,5 @@ +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import type { DaemonServer } from './transport.ts'; const DAEMON_SHUTDOWN_TIMEOUT_MS = 5_000; @@ -37,3 +39,63 @@ export async function closeDaemonServers( }), ); } + +/** What a closed daemon runtime leaves behind, by kind of resource. */ +export type ShutdownReport = { + /** Provider lease releases, by lease id. `unknown` was still in flight when the budget ran out. */ + providerReleases: { completed: string[]; pending: string[]; unknown: string[] }; + /** Device runners, by device id: `handedOff` was left running for the next daemon. */ + runners: { stopped: string[]; handedOff: string[] }; + sessions: { closed: number; failed: number }; + plugins: { disposed: string[]; failed: { provider: string; error: NormalizedError }[] }; +}; + +export type ProviderRuntimeDisposal = Readonly<{ + /** Disposes every provider runtime once; later calls return the first call's result. */ + dispose(): Promise; +}>; + +export function createProviderRuntimeDisposal( + runtimes: readonly Pick[], +): ProviderRuntimeDisposal { + let disposal: Promise | undefined; + return { + dispose: () => + (disposal ??= (async () => { + const report: ShutdownReport['plugins'] = { disposed: [], failed: [] }; + await Promise.all( + runtimes.map(async (runtime) => { + try { + await runtime.shutdown(); + report.disposed.push(runtime.provider); + } catch (error) { + report.failed.push({ provider: runtime.provider, error: normalizeError(error) }); + } + }), + ); + return report; + })()), + }; +} + +/** Undoes startup work in reverse order; a step that throws never stops the ones before it. */ +export type StartupRollback = Readonly<{ + onRollback(step: () => Promise | void): void; + run(): Promise; +}>; + +export function createStartupRollback(): StartupRollback { + const steps: Array<() => Promise | void> = []; + return { + onRollback: (step) => { + steps.push(step); + }, + run: async () => { + for (const step of steps.splice(0).reverse()) { + await Promise.resolve() + .then(step) + .catch(() => {}); + } + }, + }; +} diff --git a/packages/daemon/src/server/provider-lease-expiry.ts b/packages/daemon/src/server/provider-lease-expiry.ts index 038ef2b07d..349248ee1e 100644 --- a/packages/daemon/src/server/provider-lease-expiry.ts +++ b/packages/daemon/src/server/provider-lease-expiry.ts @@ -33,7 +33,15 @@ export type ExpiredProviderLeaseReleaser = { beginShutdown: () => void; release: (lease: DeviceLease) => Promise; retryPending: () => Promise; - drain: (timeoutMs: number) => Promise<{ pending: DeviceLease[]; released: DeviceLease[] }>; + /** + * `pending` holds every lease still unreleased; `unknown` is the part of it whose release was + * still in flight when the budget ran out, so its outcome is not known. + */ + drain: (timeoutMs: number) => Promise<{ + pending: DeviceLease[]; + unknown: DeviceLease[]; + released: DeviceLease[]; + }>; shutdown: () => void; }; @@ -206,6 +214,14 @@ export function createExpiredProviderLeaseReleaser(options: { await Promise.race([retryPending(), sleep(Math.max(0, timeoutMs))]); const result = { pending: [...pendingLiveLeases.values(), ...pendingRecoveryLeases.values()], + unknown: [ + ...[...pendingLiveLeases.values()].filter((lease) => + activeReleaseAttempts.has(`live:${lease.leaseId}`), + ), + ...[...pendingRecoveryLeases.values()].filter((lease) => + activeReleaseAttempts.has(`recovery:${lease.leaseId}`), + ), + ], released: [...shutdownReleasedLeases.values()], }; trackingShutdownReleases = false; diff --git a/packages/platform-apple/src/runner/runner-adoption.ts b/packages/platform-apple/src/runner/runner-adoption.ts index 3048c2a6a7..e90af5285b 100644 --- a/packages/platform-apple/src/runner/runner-adoption.ts +++ b/packages/platform-apple/src/runner/runner-adoption.ts @@ -83,9 +83,9 @@ export function isIosRunnerDetachEnabled(env: NodeJS.ProcessEnv = process.env): export async function detachRunnerSessionsForShutdown( sessions: ReadonlyMap, onDetached: (deviceId: string, session: RunnerSession) => void, -): Promise { - if (!isIosRunnerDetachEnabled()) return 0; - let detached = 0; +): Promise { + if (!isIosRunnerDetachEnabled()) return []; + const detached: string[] = []; for (const [deviceId, session] of sessions) { const outcome = detachRunnerSessionForShutdown(session); if (!outcome.detached) { @@ -106,7 +106,7 @@ export async function detachRunnerSessionsForShutdown( }); continue; } - detached += 1; + detached.push(deviceId); onDetached(deviceId, session); emitDiagnostic({ level: 'info', diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 16b00c4043..87b8bfb405 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -771,13 +771,15 @@ export async function abortAllIosRunnerSessions(): Promise { // The detach decision itself lives in runner-adoption.ts beside the adoption it hands off to // (#2681); detached active generations leave the registry. Idle retained generations stay // owned until shutdown stops them. -export async function detachIosRunnerSessionsForShutdown(): Promise { +export async function detachIosRunnerSessionsForShutdown(): Promise { return await detachRunnerSessionsForShutdown(runnerSessions, (deviceId) => { runnerSessions.delete(deviceId); }); } -export async function stopAllIosRunnerSessions(): Promise { +/** Stops every remaining runner and names the devices it owned. */ +export async function stopAllIosRunnerSessions(): Promise { + const owned = [...runnerSessions.keys()]; // This is the daemon's teardown: the fence spans the whole sweep, not just the abort, so a // start that began while the per-session stops ran meets a closed admission at its next gate // instead of answering the final prep sweep with another build (#3220). @@ -791,6 +793,7 @@ export async function stopAllIosRunnerSessions(): Promise { }), ); await stopRunnerPrepProcesses(); + return owned; } finally { settleFences(); } diff --git a/src/__tests__/test-utils/public-daemon-runtime.ts b/src/__tests__/test-utils/public-daemon-runtime.ts index ad8c87897c..6e8c3fcf02 100644 --- a/src/__tests__/test-utils/public-daemon-runtime.ts +++ b/src/__tests__/test-utils/public-daemon-runtime.ts @@ -52,8 +52,6 @@ export async function startPublicDaemonRuntime( AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -74,12 +72,12 @@ export async function startPublicDaemonRuntime( baseUrl: `http://127.0.0.1:${port}`, runtime, close: async () => { - await runtime.shutdown(); + await runtime.close(); fs.rmSync(stateDir, { recursive: true, force: true }); }, }; } catch (error) { - await runtime.shutdown().catch(() => {}); + await runtime.close().catch(() => {}); if (ownsStateDir) fs.rmSync(stateDir, { recursive: true, force: true }); throw error; } diff --git a/src/daemon-composition-credentials.test.ts b/src/daemon-composition-credentials.test.ts index 820fdc081d..348a278a69 100644 --- a/src/daemon-composition-credentials.test.ts +++ b/src/daemon-composition-credentials.test.ts @@ -14,8 +14,8 @@ vi.mock('./platform-runtime.ts', () => ({ createPlatformRuntimeGateway: () => ({ applicationLifecycle: { recoverStartupResources: async () => {}, - detachForDaemonShutdown: async () => {}, - finalizeDaemonShutdown: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], }, inspectFacts: async () => { throw new Error('unused'); @@ -46,8 +46,6 @@ test('daemon startup compares lease credentials with its own startup environment AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: createDaemonRuntimeDeps, @@ -81,6 +79,6 @@ test('daemon startup compares lease credentials with its own startup environment ); expect(await allocate(daemonEnv)).not.toContain('provider-credentials-changed'); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); diff --git a/src/daemon-composition.test.ts b/src/daemon-composition.test.ts index db7617016b..97d10b6fe2 100644 --- a/src/daemon-composition.test.ts +++ b/src/daemon-composition.test.ts @@ -17,8 +17,8 @@ vi.mock('./platform-runtime.ts', () => ({ createPlatformRuntimeGateway: () => ({ applicationLifecycle: { recoverStartupResources: async () => {}, - detachForDaemonShutdown: async () => {}, - finalizeDaemonShutdown: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], }, inspectFacts: async () => { throw new Error('unused'); diff --git a/src/daemon-entrypoint.test.ts b/src/daemon-entrypoint.test.ts index e28372b631..1379348e69 100644 --- a/src/daemon-entrypoint.test.ts +++ b/src/daemon-entrypoint.test.ts @@ -78,7 +78,6 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a const paths = resolveDaemonPaths(stateDir); const stdout: string[] = []; const stderr: string[] = []; - let exitCode: number | undefined; try { const runtime = await startDaemonRuntime({ @@ -90,10 +89,6 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: (chunk) => stderr.push(chunk) }, stdout: { write: (chunk) => stdout.push(chunk) }, }); @@ -111,9 +106,7 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a assert.ok(fs.existsSync(paths.lockPath), 'daemon lock should be held while runtime is active'); await waitForHttpOk(`http://127.0.0.1:${runtime?.httpPort}/health`, 2_000); - await runtime?.shutdown(); - - assert.equal(exitCode, 0); + await runtime?.close(); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -125,7 +118,6 @@ test('daemon runtime publishes dual transport metadata', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-runtime-dual-'); const paths = resolveDaemonPaths(stateDir); const stdout: string[] = []; - let exitCode: number | undefined; try { const runtime = await startDaemonRuntime({ @@ -137,10 +129,6 @@ test('daemon runtime publishes dual transport metadata', async () => { AGENT_DEVICE_DAEMON_SERVER_MODE: 'dual', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: (chunk) => stdout.push(chunk) }, }); @@ -156,9 +144,7 @@ test('daemon runtime publishes dual transport metadata', async () => { assert.equal(info.transport, 'dual'); await waitForHttpOk(`http://127.0.0.1:${runtime?.httpPort}/health`, 2_000); - await runtime?.shutdown(); - - assert.equal(exitCode, 0); + await runtime?.close(); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -193,8 +179,6 @@ test('daemon rejects unowned cloud artifacts over RPC', async () => { BROWSERSTACK_SESSION_DETAILS_ENDPOINT: `http://127.0.0.1:${providerPort}/sessions`, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -226,7 +210,7 @@ test('daemon rejects unowned cloud artifacts over RPC', async () => { assert.deepEqual(body.error?.data?.details, { reason: 'PROVIDER_SESSION_NOT_OWNED' }); assert.deepEqual(providerRequests, []); } finally { - await runtime?.shutdown(); + await runtime?.close(); await closeLoopbackServer(providerServer); fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -290,7 +274,6 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', const claimsDir = mkdtempForTestSync('agent-device-daemon-prune-claims-'); const paths = resolveDaemonPaths(stateDir); const deviceKey = 'local:android:none:prune-log'; - let exitCode: number | undefined; try { fs.writeFileSync( @@ -320,10 +303,6 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', AGENT_DEVICE_CLAIMS_DIR: claimsDir, }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -335,8 +314,7 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', assert.match(log, /"phase":"device_claim_reconcile"/); assert.match(log, /"reconciled":1/); - await runtime?.shutdown(); - assert.equal(exitCode, 0); + await runtime?.close(); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); fs.rmSync(claimsDir, { recursive: true, force: true }); @@ -444,8 +422,6 @@ test('startup sweep settles a foreign dead owner without touching a live same-na AGENT_DEVICE_CLAIMS_DIR: claimsDir, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -474,7 +450,7 @@ test('startup sweep settles a foreign dead owner without touching a live same-na "the dead owner's orphaned recorder should be terminated by recovery", ); } finally { - await runtime?.shutdown().catch(() => {}); + await runtime?.close().catch(() => {}); try { orphanRecorder.kill('SIGKILL'); } catch {} diff --git a/src/daemon.ts b/src/daemon.ts index 264b1de596..12a39b535d 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -1,16 +1,43 @@ import { asAppError } from '@agent-device/kernel/errors'; -import { startDaemonRuntime } from '@agent-device/daemon'; +import { daemonStartupExitCode, startDaemonRuntime } from '@agent-device/daemon'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; +// The executable owns the process: signals, fatal-error handlers and the exit code. The runtime +// only reports when it has closed, whoever closed it. // The root's composition enters as a factory behind a function-scoped `await import`: the server // calls it before it acquires the registration, so startup order is unchanged, and the eager // closure of this entry stays exactly the operational server's (ADR 0019). -void startDaemonRuntime({ - host: captureHostEnvironment(process), - composeDeps: async (input) => - (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input), -}).catch((error) => { - const appErr = asAppError(error); - process.stderr.write(`Daemon error: ${appErr.message}\n`); - process.exit(1); +async function runDaemon(): Promise { + const runtime = await startDaemonRuntime({ + host: captureHostEnvironment(process), + composeDeps: async (input) => + (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input), + }); + let exitCode = 0; + const closeAndExit = (code: number, cause?: unknown): void => { + exitCode = Math.max(exitCode, code); + void runtime.close({ cause }).catch(() => { + exitCode = 1; + }); + }; + const failFatally = (error: unknown): void => { + process.stderr.write(`Daemon error: ${asAppError(error).message}\n`); + closeAndExit(1, error); + }; + process.on('SIGINT', () => closeAndExit(0)); + process.on('SIGTERM', () => closeAndExit(0)); + process.on('SIGHUP', () => closeAndExit(0)); + process.on('uncaughtException', failFatally); + process.on('unhandledRejection', (reason) => + failFatally(reason instanceof Error ? reason : new Error(String(reason))), + ); + await runtime.closed.catch(() => { + exitCode = 1; + }); + process.exit(exitCode); +} + +runDaemon().catch((error) => { + process.stderr.write(`Daemon error: ${asAppError(error).message}\n`); + process.exit(daemonStartupExitCode(error)); }); diff --git a/src/platform-runtime-apple-application-tools.ts b/src/platform-runtime-apple-application-tools.ts index 35985fe3b0..b546344a86 100644 --- a/src/platform-runtime-apple-application-tools.ts +++ b/src/platform-runtime-apple-application-tools.ts @@ -114,11 +114,11 @@ export function createAppleApplicationTools(): AppleApplicationTools { dismissCloseAlerts: async (device, input) => await dismissMacOsCloseAlerts(device, input), detachRunnerSessionsForShutdown: async () => { const { detachIosRunnerSessionsForShutdown } = await loadRunnerOperations(); - await detachIosRunnerSessionsForShutdown(); + return await detachIosRunnerSessionsForShutdown(); }, finalizeRunnerSessionsForShutdown: async () => { const { stopAllIosRunnerSessions } = await loadRunnerOperations(); - await stopAllIosRunnerSessions(); + return await stopAllIosRunnerSessions(); }, }); } diff --git a/src/platform-runtime-application-resources.test.ts b/src/platform-runtime-application-resources.test.ts index 4ad3eee04e..ea728b5cdf 100644 --- a/src/platform-runtime-application-resources.test.ts +++ b/src/platform-runtime-application-resources.test.ts @@ -43,8 +43,8 @@ test('startup recovery reaches Android mechanics only after validated marker evi }); test('daemon shutdown phases delegate to the Apple runner owner', async () => { - const detachRunnerSessionsForShutdown = vi.fn(async () => {}); - const finalizeRunnerSessionsForShutdown = vi.fn(async () => {}); + const detachRunnerSessionsForShutdown = vi.fn(async () => ['ios-sim']); + const finalizeRunnerSessionsForShutdown = vi.fn(async () => ['ios-sim']); const resources = lifecycle({ hasTestImeRecoveryEvidence: async () => false, recoverTestImeStartup: async () => {}, diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index 54feedf435..867cb313ea 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -74,8 +74,8 @@ export function createComposedPlatformRuntimeGateway(options: { }; // Which durable resources exist is the host's composition problem; the gateway only owns the // once-per-process shape of each phase and keeps the host load lazy. - let lifecycleDetach: Promise | undefined; - let lifecycleFinalize: Promise | undefined; + let lifecycleDetach: Promise | undefined; + let lifecycleFinalize: Promise | undefined; const applicationLifecycle = Object.freeze({ recoverStartupResources: async (input: Readonly<{ stateDir: string }>) => { await runStartupRecoveryFence(input.stateDir, async () => { @@ -86,13 +86,13 @@ export function createComposedPlatformRuntimeGateway(options: { lifecycleDetach ??= loadHost().then( async (host) => await host.applicationResources.detachForDaemonShutdown(), ); - await lifecycleDetach; + return await lifecycleDetach; }, finalizeDaemonShutdown: async () => { lifecycleFinalize ??= loadHost().then( async (host) => await host.applicationResources.finalizeDaemonShutdown(), ); - await lifecycleFinalize; + return await lifecycleFinalize; }, }); const registerOwner = (owner: PlatformRuntimeOwner) => { diff --git a/src/provider-device-runtimes.test.ts b/src/provider-device-runtimes.test.ts index 8ef20ecdb5..7269651fd5 100644 --- a/src/provider-device-runtimes.test.ts +++ b/src/provider-device-runtimes.test.ts @@ -73,7 +73,7 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest interval: 10, timeout: 2_000, }); - await firstDaemon.runtime.shutdown(); + await firstDaemon.runtime.close(); assert.ok(limrun.iosDisconnect.mock.calls.length > 0); const pendingPath = path.join(stateDir, 'expired-provider-leases.json'); @@ -100,7 +100,7 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest if (restartedDaemon) { await restartedDaemon.close(); } else { - await firstDaemon?.runtime.shutdown(); + await firstDaemon?.runtime.close(); fs.rmSync(stateDir, { recursive: true, force: true }); } } From 578983831d0f3e6df555836f7ec8da6321668150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 23:19:10 +0200 Subject: [PATCH 02/11] feat(daemon): per-runtime state for admission, resolution, artifacts, uploads and requests Provider admission, interactor resolution, artifact and upload tracking, resumable uploads, materialized paths, session-open locks and request cancellation are owned by the runtime instance instead of module scope. Co-Authored-By: Claude Sonnet 5.5 --- .../daemon/src/core/interactor-resolution.ts | 21 +++- .../src/core/provider-device-admission.ts | 21 +++- .../daemon/src/resources/artifact-tracking.ts | 55 ++++++--- .../http/daemon-runtime-artifacts.test.ts | 17 +-- .../http/daemon-runtime-composition.test.ts | 12 +- .../http/daemon-runtime-isolation.test.ts | 104 ++++++++++++++++++ .../daemon/src/server/http/daemon-runtime.ts | 71 +++++++++--- .../daemon/src/server/http/http-server.ts | 18 ++- packages/daemon/src/server/http/transport.ts | 14 ++- .../daemon/src/server/resumable-upload.ts | 46 ++++++-- .../sessions/materialized-path-registry.ts | 29 ++++- .../internal/session-open.ts | 14 ++- .../host-kit/src/internal/request-cancel.ts | 66 ++++++++--- packages/host-kit/src/request.ts | 2 + 14 files changed, 400 insertions(+), 90 deletions(-) create mode 100644 packages/daemon/src/server/http/daemon-runtime-isolation.test.ts diff --git a/packages/daemon/src/core/interactor-resolution.ts b/packages/daemon/src/core/interactor-resolution.ts index 2584685635..aea563e018 100644 --- a/packages/daemon/src/core/interactor-resolution.ts +++ b/packages/daemon/src/core/interactor-resolution.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import type { Interactor, RunnerContext } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -27,7 +29,20 @@ const NO_INTERACTOR_RESOLUTION: InteractorResolution = { }, }; -let installedInteractorResolution: InteractorResolution = NO_INTERACTOR_RESOLUTION; +type InteractorResolutionState = { resolution: InteractorResolution }; + +const interactorResolutionStorage = new AsyncLocalStorage(); +const standaloneInteractorResolution: InteractorResolutionState = { + resolution: NO_INTERACTOR_RESOLUTION, +}; +const interactorResolutionState = (): InteractorResolutionState => + interactorResolutionStorage.getStore() ?? standaloneInteractorResolution; + +/** The resolution one daemon runtime owns; the runtime installs its composition's into it. */ +export function createInteractorResolutionRuntimeState(): RuntimeStateHandle { + const owned: InteractorResolutionState = { resolution: NO_INTERACTOR_RESOLUTION }; + return { run: (fn) => interactorResolutionStorage.run(owned, fn), dispose: () => {} }; +} /** * Root composition's installation point, beside the provider-device admission that shares the @@ -35,9 +50,9 @@ let installedInteractorResolution: InteractorResolution = NO_INTERACTOR_RESOLUTI * the module root composition imports, and the daemon never loads it itself. */ export function installInteractorResolution(resolution: InteractorResolution): void { - installedInteractorResolution = resolution; + interactorResolutionState().resolution = resolution; } export function interactorResolution(): InteractorResolution { - return installedInteractorResolution; + return interactorResolutionState().resolution; } diff --git a/packages/daemon/src/core/provider-device-admission.ts b/packages/daemon/src/core/provider-device-admission.ts index bae7032244..8185f6dc9a 100644 --- a/packages/daemon/src/core/provider-device-admission.ts +++ b/packages/daemon/src/core/provider-device-admission.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import type { DeviceInfo } from '@agent-device/kernel/device'; /** @@ -18,7 +20,20 @@ const NO_PROVIDER_DEVICE_ADMISSION: ProviderDeviceAdmission = { isActive: () => false, }; -let installedProviderDeviceAdmission: ProviderDeviceAdmission = NO_PROVIDER_DEVICE_ADMISSION; +type ProviderAdmissionState = { admission: ProviderDeviceAdmission }; + +const providerAdmissionStorage = new AsyncLocalStorage(); +const standaloneProviderAdmission: ProviderAdmissionState = { + admission: NO_PROVIDER_DEVICE_ADMISSION, +}; +const providerAdmissionState = (): ProviderAdmissionState => + providerAdmissionStorage.getStore() ?? standaloneProviderAdmission; + +/** The admission one daemon runtime owns; the runtime installs its composition's into it. */ +export function createProviderDeviceAdmissionRuntimeState(): RuntimeStateHandle { + const owned: ProviderAdmissionState = { admission: NO_PROVIDER_DEVICE_ADMISSION }; + return { run: (fn) => providerAdmissionStorage.run(owned, fn), dispose: () => {} }; +} /** * Root composition's installation point. The provider runtime scope is ambient per request, so @@ -26,9 +41,9 @@ let installedProviderDeviceAdmission: ProviderDeviceAdmission = NO_PROVIDER_DEVI * provider runtime internals itself. */ export function installProviderDeviceAdmission(admission: ProviderDeviceAdmission): void { - installedProviderDeviceAdmission = admission; + providerAdmissionState().admission = admission; } export function isActiveProviderDevice(device: DeviceInfo): boolean { - return installedProviderDeviceAdmission.isActive(device); + return providerAdmissionState().admission.isActive(device); } diff --git a/packages/daemon/src/resources/artifact-tracking.ts b/packages/daemon/src/resources/artifact-tracking.ts index efccad7590..2fcb74fdf3 100644 --- a/packages/daemon/src/resources/artifact-tracking.ts +++ b/packages/daemon/src/resources/artifact-tracking.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; @@ -44,7 +46,36 @@ type ArtifactEntry = { timer: ReturnType; }; -const pendingArtifacts = new Map(); +type ArtifactTrackingState = { + artifacts: Map; + uploads: Map; +}; + +const artifactTrackingStorage = new AsyncLocalStorage(); +const createArtifactTrackingState = (): ArtifactTrackingState => ({ + artifacts: new Map(), + uploads: new Map(), +}); +const standaloneArtifactTracking = createArtifactTrackingState(); +const artifactTracking = (): ArtifactTrackingState => + artifactTrackingStorage.getStore() ?? standaloneArtifactTracking; +const pendingArtifacts = () => artifactTracking().artifacts; +const pendingUploads = () => artifactTracking().uploads; + +/** The artifacts and uploads one daemon runtime tracks; disposing it stops their expiry timers. */ +export function createArtifactTrackingRuntimeState(): RuntimeStateHandle { + const owned = createArtifactTrackingState(); + return { + run: (fn) => artifactTrackingStorage.run(owned, fn), + dispose: () => { + for (const entry of [...owned.artifacts.values(), ...owned.uploads.values()]) { + clearTimeout(entry.timer); + } + owned.artifacts.clear(); + owned.uploads.clear(); + }, + }; +} /** What a finished request registers for each artifact it hands back to the client. */ export type DownloadableArtifactRegistration = { @@ -73,7 +104,7 @@ export function trackDownloadableArtifact( cleanupDownloadableArtifact(artifactId); }, ARTIFACT_CLEANUP_TIMEOUT_MS); timer.unref(); - pendingArtifacts.set(artifactId, { + pendingArtifacts().set(artifactId, { artifactPath: params.artifactPath, tenantId: params.tenantId, artifactType: params.artifactType, @@ -105,7 +136,7 @@ export async function listDownloadableArtifacts( tenantId?: string, ): Promise { const artifacts: DaemonArtifactInventoryEntry[] = []; - for (const [id, entry] of pendingArtifacts) { + for (const [id, entry] of pendingArtifacts()) { if (!canReadArtifact(entry, tenantId)) continue; let stat: fs.Stats; try { @@ -148,7 +179,7 @@ function readDownloadableArtifactEntry( tenantId: string | undefined, ): { entry: ArtifactEntry; stat: fs.Stats } { const entry = requireTenantOwnedEntry( - pendingArtifacts, + pendingArtifacts(), artifactId, tenantId, DOWNLOADABLE_ARTIFACT_RESOURCE, @@ -249,7 +280,7 @@ async function createDirectoryArchive( sizeBytes: fs.statSync(archivePath).size, }; // The artifact may be consumed or expire while tar is still archiving. - if (pendingArtifacts.get(artifactId) !== entry) { + if (pendingArtifacts().get(artifactId) !== entry) { fs.rmSync(tempDir, { recursive: true, force: true }); throw expiredTenantOwnedEntryError(DOWNLOADABLE_ARTIFACT_RESOURCE, artifactId); } @@ -294,10 +325,10 @@ function hasUnsafeFilenameCharacter(fileName: string): boolean { } export function cleanupDownloadableArtifact(artifactId: string): void { - const entry = pendingArtifacts.get(artifactId); + const entry = pendingArtifacts().get(artifactId); if (!entry) return; clearTimeout(entry.timer); - pendingArtifacts.delete(artifactId); + pendingArtifacts().delete(artifactId); cleanupDirectoryArchive(entry); if (!entry.deleteAfterDownload) return; try { @@ -334,8 +365,6 @@ type UploadEntry = { timer: ReturnType; }; -const pendingUploads = new Map(); - export function trackUploadedArtifact(params: { artifactPath: string; tempDir: string; @@ -346,7 +375,7 @@ export function trackUploadedArtifact(params: { cleanupUploadedArtifact(uploadId); }, UPLOAD_CLEANUP_TIMEOUT_MS); timer.unref(); - pendingUploads.set(uploadId, { + pendingUploads().set(uploadId, { artifactPath: params.artifactPath, tempDir: params.tempDir, tenantId: params.tenantId, @@ -357,7 +386,7 @@ export function trackUploadedArtifact(params: { export function prepareUploadedArtifact(uploadId: string, tenantId?: string): string { const entry = requireTenantOwnedEntry( - pendingUploads, + pendingUploads(), uploadId, tenantId, UPLOADED_ARTIFACT_RESOURCE, @@ -367,9 +396,9 @@ export function prepareUploadedArtifact(uploadId: string, tenantId?: string): st } export function cleanupUploadedArtifact(uploadId: string): void { - const entry = pendingUploads.get(uploadId); + const entry = pendingUploads().get(uploadId); if (!entry) return; clearTimeout(entry.timer); - pendingUploads.delete(uploadId); + pendingUploads().delete(uploadId); fs.rmSync(entry.tempDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts index b7e1f1e26a..16560207b9 100644 --- a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts @@ -13,11 +13,6 @@ async function downloadTrackedArtifact(env: Record, awaitConsump const stateDir = mkdtempForTestSync('agent-device-daemon-runtime-artifacts-'); const artifactPath = path.join(stateDir, 'runtime-artifact.txt'); fs.writeFileSync(artifactPath, 'runtime-artifact'); - const artifactId = trackDownloadableArtifact({ - artifactPath, - artifactType: 'runtime-artifact', - fileName: 'runtime-artifact.txt', - }); const runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { @@ -32,8 +27,14 @@ async function downloadTrackedArtifact(env: Record, awaitConsump stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture(), }); + const artifactId = runtime.run(() => + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'runtime-artifact.txt', + }), + ); try { - expect(runtime).not.toBeNull(); const response = await fetch( `http://127.0.0.1:${runtime?.httpPort}/artifacts/${encodeURIComponent(artifactId)}`, { headers: { authorization: `Bearer ${runtime?.token}`, connection: 'close' } }, @@ -46,8 +47,8 @@ async function downloadTrackedArtifact(env: Record, awaitConsump } return { status: response.status, body, onDisk: fs.existsSync(artifactPath) }; } finally { - await runtime?.close(); - cleanupDownloadableArtifact(artifactId); + runtime.run(() => cleanupDownloadableArtifact(artifactId)); + await runtime.close(); } } diff --git a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts index aacf50c595..366f8f225e 100644 --- a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts @@ -14,10 +14,7 @@ import { WEB_DESKTOP_DEVICE, createTestHostEnvironment, } from '@agent-device/test-kit'; -import { - installInteractorResolution, - interactorResolution, -} from '../../core/interactor-resolution.ts'; +import { interactorResolution } from '../../core/interactor-resolution.ts'; import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; import { daemonStartupExitCode, @@ -46,7 +43,6 @@ vi.mock('../../core/session-store.ts', async (importOriginal) => { test('daemon startup installs the interactor resolution the composition supplied', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-interactor-composition-'); const composed = Object.freeze({ resolve: vi.fn(async () => ({}) as never) }); - const previousResolution = interactorResolution(); let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ @@ -64,12 +60,12 @@ test('daemon startup installs the interactor resolution the composition supplied }); expect(runtime).not.toBeNull(); - expect(interactorResolution()).toBe(composed); - await interactorResolution().resolve(IOS_SIMULATOR, {}); + const resolution = runtime.run(() => interactorResolution()); + expect(resolution).toBe(composed); + await resolution.resolve(IOS_SIMULATOR, {}); expect(composed.resolve).toHaveBeenCalledOnce(); } finally { await runtime?.close(); - installInteractorResolution(previousResolution); fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts new file mode 100644 index 0000000000..b00d232616 --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts @@ -0,0 +1,104 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { interactorResolution } from '../../core/interactor-resolution.ts'; +import { isActiveProviderDevice } from '../../core/provider-device-admission.ts'; +import { trackDownloadableArtifact } from '../../resources/artifact-tracking.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +async function startIsolatedRuntime(label: string, interactorTag: string, providerActive: boolean) { + const stateDir = mkdtempForTestSync(`agent-device-daemon-isolation-${label}-`); + const runtime = await startDaemonRuntime({ + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }), + stderr: { write: () => {} }, + stdout: { write: () => {} }, + composeDeps: daemonRuntimeDepsFixture({ + interactorResolution: Object.freeze({ + resolve: async () => { + throw new Error(interactorTag); + }, + }), + providerDeviceAdmission: Object.freeze({ isActive: () => providerActive }), + }), + }); + const { httpPort, token } = JSON.parse( + fs.readFileSync(resolveDaemonPaths(stateDir).infoPath, 'utf8'), + ) as { httpPort: number; token: string }; + return { runtime, stateDir, httpPort, token }; +} + +async function callSessionList(httpPort: number, token: string, requestId: string) { + const response = await fetch(`http://127.0.0.1:${httpPort}/rpc`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method: 'agent_device.command', + params: { command: 'session_list', meta: { requestId } }, + }), + }); + return (await response.json()) as { result?: unknown; error?: unknown }; +} + +test('two runtimes in one process serve, close and keep their state independently', async () => { + const a = await startIsolatedRuntime('a', 'interactor-a', true); + const b = await startIsolatedRuntime('b', 'interactor-b', false); + try { + expect(a.httpPort).not.toBe(b.httpPort); + expect(path.basename(a.stateDir)).not.toBe(path.basename(b.stateDir)); + + // Each runtime answers requests with its own token and its own state dir. + expect((await callSessionList(a.httpPort, a.token, 'iso-a-1')).error).toBeUndefined(); + expect((await callSessionList(b.httpPort, b.token, 'iso-b-1')).error).toBeUndefined(); + + // Per-runtime state: what one installs or tracks is invisible to the other. + expect(a.runtime.run(() => isActiveProviderDevice({} as never))).toBe(true); + expect(b.runtime.run(() => isActiveProviderDevice({} as never))).toBe(false); + await expect( + a.runtime.run(() => interactorResolution().resolve({} as never, {})), + ).rejects.toThrow('interactor-a'); + await expect( + b.runtime.run(() => interactorResolution().resolve({} as never, {})), + ).rejects.toThrow('interactor-b'); + + const artifactPath = path.join(a.stateDir, 'artifact.txt'); + fs.writeFileSync(artifactPath, 'artifact-a'); + const artifactId = a.runtime.run(() => + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'artifact.txt', + }), + ); + const fromB = await fetch( + `http://127.0.0.1:${b.httpPort}/artifacts/${encodeURIComponent(artifactId)}`, + { headers: { authorization: `Bearer ${b.token}`, connection: 'close' } }, + ); + expect(fromB.status).not.toBe(200); + await fromB.text(); + + // Closing one runtime leaves the other serving. + const reportA = await a.runtime.close(); + expect(reportA.sessions).toEqual({ closed: 0, failed: 0 }); + await expect(callSessionList(a.httpPort, a.token, 'iso-a-2')).rejects.toThrow(); + expect((await callSessionList(b.httpPort, b.token, 'iso-b-2')).error).toBeUndefined(); + expect(b.runtime.run(() => isActiveProviderDevice({} as never))).toBe(false); + + const reportB = await b.runtime.close(); + expect(reportB.plugins).toEqual({ disposed: [], failed: [] }); + } finally { + await a.runtime.close(); + await b.runtime.close(); + } +}); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 143f8889da..f3795bb549 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -13,8 +13,14 @@ import { resolveDaemonServerMode, } from '@agent-device/daemon-contracts/daemon-resolution'; import { createDaemonHttpServer } from './http-server.ts'; -import { trackDownloadableArtifact } from '../../resources/artifact-tracking.ts'; import { + createArtifactTrackingRuntimeState, + trackDownloadableArtifact, +} from '../../resources/artifact-tracking.ts'; +import { createResumableUploadRuntimeState } from '../resumable-upload.ts'; +import { createMaterializedPathRuntimeState } from '../../sessions/materialized-path-registry.ts'; +import { + createProviderDeviceAdmissionRuntimeState, installProviderDeviceAdmission, type ProviderDeviceAdmission, } from '../../core/provider-device-admission.ts'; @@ -24,6 +30,7 @@ import { } from '@agent-device/daemon-contracts/daemon-policy-file'; import { assertDaemonPolicyAllowsCapability } from '../../core/daemon-policy.ts'; import { + createInteractorResolutionRuntimeState, installInteractorResolution, type InteractorResolution, } from '../../core/interactor-resolution.ts'; @@ -64,6 +71,10 @@ import { createAudioProbeAdmissionLedger } from '@agent-device/capture-kit/audio import { createPerfCaptureAdmissionLedger } from '@agent-device/capture-kit/perf-capture-admission-ledger'; import { createScreenRecordingAdmissionLedger } from '@agent-device/capture-kit/screen-recording-admission-ledger'; import { withCommandHost } from '@agent-device/host-kit/command'; +import { + createRequestCancelRuntimeState, + type RuntimeStateHandle, +} from '@agent-device/host-kit/request'; import { emitDiagnostic, type ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; import { @@ -218,6 +229,8 @@ export type DaemonRuntimeController = { close: (options?: { cause?: unknown }) => Promise; /** Settles once the runtime has closed, whoever closed it (an idle reap closes it unasked). */ closed: Promise; + /** Runs `fn` as this runtime, for code that serves it from outside its own callbacks. */ + run: (fn: () => T) => T; token: string; }; @@ -412,23 +425,49 @@ export async function teardownDaemonSessionForShutdown(params: { return teardownSucceeded; } +type RuntimeContext = Readonly<{ run(fn: () => T): T; dispose(): void }>; + +/** The per-runtime state of every module that keeps some, in the order it is entered. */ +function createDaemonRuntimeStates(): RuntimeStateHandle[] { + return [ + createRequestCancelRuntimeState(), + createProviderDeviceAdmissionRuntimeState(), + createInteractorResolutionRuntimeState(), + createArtifactTrackingRuntimeState(), + createResumableUploadRuntimeState(), + createMaterializedPathRuntimeState(), + ]; +} + export async function startDaemonRuntime( options: DaemonRuntimeOptions, ): Promise { - return await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { - const rollback = createStartupRollback(); - try { - return await startDaemonRuntimeOnHost(options, rollback); - } catch (error) { - await rollback.run(); - throw error; - } - }); + const states = createDaemonRuntimeStates(); + const runtimeContext: RuntimeContext = { + run: (fn) => states.reduceRight((next, state) => () => state.run(next), fn)(), + dispose: () => { + for (const state of states) state.dispose(); + }, + }; + return await runtimeContext.run( + async () => + await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { + const rollback = createStartupRollback(); + rollback.onRollback(() => runtimeContext.dispose()); + try { + return await startDaemonRuntimeOnHost(options, rollback, runtimeContext); + } catch (error) { + await rollback.run(); + throw error; + } + }), + ); } async function startDaemonRuntimeOnHost( options: DaemonRuntimeOptions, rollback: StartupRollback, + runtimeContext: RuntimeContext, ): Promise { const { host } = options; const { env } = host; @@ -681,7 +720,7 @@ async function startDaemonRuntimeOnHost( }, }; idleReap.cancel(); - const dispatch = dispatchRequest(req); + const dispatch = runtimeContext.run(() => dispatchRequest(req)); inFlightRequests.add(dispatch); try { return await dispatch; @@ -953,6 +992,7 @@ async function startDaemonRuntimeOnHost( outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, }); await deps.ownerLifecycle.clearDaemonLockConfiguration(); + runtimeContext.dispose(); const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); return { providerReleases: { @@ -980,9 +1020,11 @@ async function startDaemonRuntimeOnHost( closed.catch(() => {}); const close = (closeOptions: { cause?: unknown } = {}): Promise => { if (!closing) { - closing = withCommandHost( - { host, requestEnv: undefined }, - async () => await closeOnHost(closeOptions), + closing = runtimeContext.run(() => + withCommandHost( + { host, requestEnv: undefined }, + async () => await closeOnHost(closeOptions), + ), ); closing.then(settleClosed.resolve, settleClosed.reject); } @@ -993,6 +1035,7 @@ async function startDaemonRuntimeOnHost( httpPort, close, closed, + run: runtimeContext.run, socketPort, token, }; diff --git a/packages/daemon/src/server/http/http-server.ts b/packages/daemon/src/server/http/http-server.ts index e5e5d878f6..6259b0df7d 100644 --- a/packages/daemon/src/server/http/http-server.ts +++ b/packages/daemon/src/server/http/http-server.ts @@ -6,6 +6,7 @@ // in `http-rpc-envelope.ts`, the authorization decisions in // `http-authorization.ts`, and the RPC method vocabulary in `http-rpc-methods.ts`. import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import { AsyncResource } from 'node:async_hooks'; import http from 'node:http'; import { randomUUID } from 'node:crypto'; import type { DiagnosticsRecordRef, NormalizedError } from '@agent-device/kernel/errors'; @@ -104,7 +105,7 @@ export async function createDaemonHttpServer(options: { resolveRequestDiagnosticsPath, onUnownedRequestFailure, } = options; - return http.createServer((req, res) => { + const onRequest = AsyncResource.bind((req: http.IncomingMessage, res: http.ServerResponse) => { if (req.method === 'GET' && req.url === '/health') { res.statusCode = 200; res.setHeader('content-type', 'application/json'); @@ -212,6 +213,7 @@ export async function createDaemonHttpServer(options: { }); }); }); + return http.createServer(onRequest); } /** Accumulates one `/rpc` body and hands it to the flow once the stream ends. */ @@ -235,9 +237,12 @@ function readRpcRequestBody( } }); - req.on('end', () => { - onBody(body); - }); + req.on( + 'end', + AsyncResource.bind(() => { + onBody(body); + }), + ); } // The `/rpc` flow, in the order it runs: the envelope decides whether this body is a @@ -471,8 +476,9 @@ function beginRpcRequestLifetime(params: { }; }, watchForClientDisconnect() { - req.on('aborted', markCanceledIfResponseIncomplete); - res.on('close', markCanceledIfResponseIncomplete); + const onClientGone = AsyncResource.bind(markCanceledIfResponseIncomplete); + req.on('aborted', onClientGone); + res.on('close', onClientGone); if (req.aborted || res.destroyed) { markCanceledIfResponseIncomplete(); } diff --git a/packages/daemon/src/server/http/transport.ts b/packages/daemon/src/server/http/transport.ts index c5e22a6329..83a5dbcb0a 100644 --- a/packages/daemon/src/server/http/transport.ts +++ b/packages/daemon/src/server/http/transport.ts @@ -1,5 +1,6 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { AppError, normalizeError, createRequestCanceledError } from '@agent-device/kernel/errors'; +import { AsyncResource } from 'node:async_hooks'; import net from 'node:net'; import type { Server as HttpServer } from 'node:http'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; @@ -27,8 +28,12 @@ export type DaemonServer = (net.Server | HttpServer) & { export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer { const sockets = new Set(); const server: DaemonServer = net.createServer((socket) => { + // An accepted socket loses the runtime that is listening, so its callbacks keep it explicitly. sockets.add(socket); - socket.on('close', () => sockets.delete(socket)); + socket.on( + 'close', + AsyncResource.bind(() => sockets.delete(socket)), + ); let buffer = ''; let inFlightRequests = 0; const activeRequestIds = new Set(); @@ -52,9 +57,9 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer }); }; socket.setEncoding('utf8'); - socket.on('close', cancelInFlightRequests); - socket.on('error', cancelInFlightRequests); - socket.on('data', async (chunk) => { + socket.on('close', AsyncResource.bind(cancelInFlightRequests)); + socket.on('error', AsyncResource.bind(cancelInFlightRequests)); + const onData = AsyncResource.bind(async (chunk: string) => { const parsed = consumeTextLines(buffer, chunk); buffer = parsed.buffer; for (const line of parsed.lines) { @@ -104,6 +109,7 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer } } }); + socket.on('data', onData); }); server.destroyConnections = () => { for (const socket of sockets) { diff --git a/packages/daemon/src/server/resumable-upload.ts b/packages/daemon/src/server/resumable-upload.ts index 74a86bd2ba..fd6a8f9baa 100644 --- a/packages/daemon/src/server/resumable-upload.ts +++ b/packages/daemon/src/server/resumable-upload.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; @@ -20,8 +22,34 @@ const RESUMABLE_UPLOAD_RESOURCE: TenantOwnedResourceKind = { label: 'Upload', expiredHint: `Resumable uploads expire ${RESUMABLE_UPLOAD_CLEANUP_TIMEOUT_MS / 60_000} minutes after the last received chunk. Start the upload again from the beginning.`, }; -const RESUMABLE_UPLOADS_BY_ID = new Map(); -const RESUMABLE_UPLOADS_BY_KEY = new Map(); +type ResumableUploadState = { + byId: Map; + byKey: Map; +}; + +const resumableUploadStorage = new AsyncLocalStorage(); +const createResumableUploadState = (): ResumableUploadState => ({ + byId: new Map(), + byKey: new Map(), +}); +const standaloneResumableUploads = createResumableUploadState(); +const resumableUploads = (): ResumableUploadState => + resumableUploadStorage.getStore() ?? standaloneResumableUploads; +const RESUMABLE_UPLOADS_BY_ID = () => resumableUploads().byId; +const RESUMABLE_UPLOADS_BY_KEY = () => resumableUploads().byKey; + +/** The resumable uploads one daemon runtime holds; disposing it stops their expiry timers. */ +export function createResumableUploadRuntimeState(): RuntimeStateHandle { + const owned = createResumableUploadState(); + return { + run: (fn) => resumableUploadStorage.run(owned, fn), + dispose: () => { + for (const entry of owned.byId.values()) clearTimeout(entry.timer); + owned.byId.clear(); + owned.byKey.clear(); + }, + }; +} export type BeginResumableUploadOptions = { baseUrl: string; @@ -62,8 +90,8 @@ export function beginResumableUpload(options: BeginResumableUploadOptions): { } { validateResumableUploadOptions(options); const key = buildResumableUploadKey(options); - const existingId = RESUMABLE_UPLOADS_BY_KEY.get(key); - const existing = existingId ? RESUMABLE_UPLOADS_BY_ID.get(existingId) : undefined; + const existingId = RESUMABLE_UPLOADS_BY_KEY().get(key); + const existing = existingId ? RESUMABLE_UPLOADS_BY_ID().get(existingId) : undefined; const entry = existing ?? createResumableUploadEntry(options, key); return { uploadId: entry.id, @@ -154,8 +182,8 @@ function createResumableUploadEntry( closed: false, finalized: false, }; - RESUMABLE_UPLOADS_BY_ID.set(id, entry); - RESUMABLE_UPLOADS_BY_KEY.set(key, id); + RESUMABLE_UPLOADS_BY_ID().set(id, entry); + RESUMABLE_UPLOADS_BY_KEY().set(key, id); refreshResumableUploadTimer(entry); return entry; } @@ -258,8 +286,8 @@ async function terminateEntry(entry: ResumableUploadEntry): Promise { function closeEntry(entry: ResumableUploadEntry): void { entry.closed = true; if (entry.timer) clearTimeout(entry.timer); - RESUMABLE_UPLOADS_BY_ID.delete(entry.id); - RESUMABLE_UPLOADS_BY_KEY.delete(entry.key); + RESUMABLE_UPLOADS_BY_ID().delete(entry.id); + RESUMABLE_UPLOADS_BY_KEY().delete(entry.key); } function requireResumableUpload( @@ -267,7 +295,7 @@ function requireResumableUpload( tenantId: string | undefined, ): ResumableUploadEntry { return requireTenantOwnedEntry( - RESUMABLE_UPLOADS_BY_ID, + RESUMABLE_UPLOADS_BY_ID(), uploadId, tenantId, RESUMABLE_UPLOAD_RESOURCE, diff --git a/packages/daemon/src/sessions/materialized-path-registry.ts b/packages/daemon/src/sessions/materialized-path-registry.ts index edc4d9e207..57494f7481 100644 --- a/packages/daemon/src/sessions/materialized-path-registry.ts +++ b/packages/daemon/src/sessions/materialized-path-registry.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import crypto from 'node:crypto'; import fs from 'node:fs/promises'; import os from 'node:os'; @@ -33,7 +35,22 @@ export type RetainedMaterializedPaths = { expiresAt: string; }; -const retainedPaths = new Map(); +const retainedPathsStorage = new AsyncLocalStorage>(); +const standaloneRetainedPaths = new Map(); +const retainedPaths = (): Map => + retainedPathsStorage.getStore() ?? standaloneRetainedPaths; + +/** The materialized paths one daemon runtime retains; disposing it stops their expiry timers. */ +export function createMaterializedPathRuntimeState(): RuntimeStateHandle { + const owned = new Map(); + return { + run: (fn) => retainedPathsStorage.run(owned, fn), + dispose: () => { + for (const entry of owned.values()) clearTimeout(entry.timer); + owned.clear(); + }, + }; +} export async function retainMaterializedPaths(params: { installablePath: string; @@ -57,7 +74,7 @@ export async function retainMaterializedPaths(params: { const timer = setTimeout(() => { void expireRetainedMaterializedPaths(materializationId); }, ttlMs); - retainedPaths.set(materializationId, { + retainedPaths().set(materializationId, { rootPath, installablePath: retainedInstallablePath, archivePath: retainedArchivePath, @@ -83,7 +100,7 @@ export async function cleanupRetainedMaterializedPaths( tenantId?: string, ): Promise { const entry = requireTenantOwnedEntry( - retainedPaths, + retainedPaths(), materializationId, tenantId, MATERIALIZED_PATHS_RESOURCE, @@ -94,7 +111,7 @@ export async function cleanupRetainedMaterializedPaths( export async function cleanupRetainedMaterializedPathsForSession( sessionName: string, ): Promise { - const matchingEntries = Array.from(retainedPaths.entries()).filter( + const matchingEntries = Array.from(retainedPaths().entries()).filter( ([, entry]) => entry.sessionName === sessionName, ); await Promise.all( @@ -107,7 +124,7 @@ export async function cleanupRetainedMaterializedPathsForSession( // Daemon-internal expiry: skips the tenant check (the timer has no tenant // context) and must never reject — the caller is a bare setTimeout. async function expireRetainedMaterializedPaths(materializationId: string): Promise { - const entry = retainedPaths.get(materializationId); + const entry = retainedPaths().get(materializationId); if (!entry) return; try { await removeRetainedEntry(materializationId, entry); @@ -118,7 +135,7 @@ async function expireRetainedMaterializedPaths(materializationId: string): Promi async function removeRetainedEntry(materializationId: string, entry: RetainedEntry): Promise { clearTimeout(entry.timer); - retainedPaths.delete(materializationId); + retainedPaths().delete(materializationId); await fs.rm(entry.rootPath, { recursive: true, force: true }); } diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts index 03be15c5e5..f370fe7869 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts @@ -65,7 +65,17 @@ export type SessionOpenCommandInput = Readonly<{ reconcileOrphanedDeviceClaim: DeviceClaimReconciler; }>; -const firstSessionOpenLocks = new Map>(); +// One lock table per session store, which is to say per daemon runtime. +const firstSessionOpenLocksByStore = new WeakMap>>(); + +function firstSessionOpenLocks(sessionStore: SessionStore): Map> { + let locks = firstSessionOpenLocksByStore.get(sessionStore); + if (!locks) { + locks = new Map(); + firstSessionOpenLocksByStore.set(sessionStore, locks); + } + return locks; +} type OpenRuntimeAdmission = | Readonly<{ @@ -343,7 +353,7 @@ async function handleOpenCommand(params: SessionOpenCommandInput): Promise await openNewSessionWithDeviceClaim({ diff --git a/packages/host-kit/src/internal/request-cancel.ts b/packages/host-kit/src/internal/request-cancel.ts index 6d0b65516c..091cfa326a 100644 --- a/packages/host-kit/src/internal/request-cancel.ts +++ b/packages/host-kit/src/internal/request-cancel.ts @@ -1,7 +1,45 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; -const canceledRequestIds = new Set(); -const requestAbortControllers = new Map(); +/** + * What one daemon runtime owns of a kind: the module keeps its state in an async-local slot the + * runtime enters, so two runtimes in one process never share it. Outside a runtime (a client + * process, a unit test) the module's own standalone state answers. + */ +export type RuntimeStateHandle = Readonly<{ + /** Runs `fn` as the runtime that owns this state. */ + run(fn: () => T): T; + /** Releases what the state holds (timers, handles). */ + dispose(): void; +}>; + +type RequestCancelState = { + canceledRequestIds: Set; + requestAbortControllers: Map; +}; + +const createRequestCancelState = (): RequestCancelState => ({ + canceledRequestIds: new Set(), + requestAbortControllers: new Map(), +}); +const requestCancelStorage = new AsyncLocalStorage(); +const standaloneRequestCancelState = createRequestCancelState(); +const requestCancelState = (): RequestCancelState => + requestCancelStorage.getStore() ?? standaloneRequestCancelState; +const canceledRequestIds = () => requestCancelState().canceledRequestIds; +const requestAbortControllers = () => requestCancelState().requestAbortControllers; + +export function createRequestCancelRuntimeState(): RuntimeStateHandle { + const owned = createRequestCancelState(); + return { + run: (fn) => requestCancelStorage.run(owned, fn), + dispose: () => { + for (const controller of owned.requestAbortControllers.values()) controller.abort(); + owned.requestAbortControllers.clear(); + owned.canceledRequestIds.clear(); + }, + }; +} export type RequestAbortRegistration = { requestId: string; @@ -55,16 +93,16 @@ export function registerRequestAbort( requestId: string | undefined, ): RequestAbortRegistration | undefined { if (!requestId) return undefined; - if (requestAbortControllers.has(requestId)) { + if (requestAbortControllers().has(requestId)) { throw new AppError('INVALID_ARGS', `Request ID is already in flight: ${requestId}`, { requestId, reason: 'duplicate_request_id', }); } - evictOldestEntries(requestAbortControllers); + evictOldestEntries(requestAbortControllers()); const controller = new AbortController(); - requestAbortControllers.set(requestId, controller); - if (canceledRequestIds.has(requestId)) { + requestAbortControllers().set(requestId, controller); + if (canceledRequestIds().has(requestId)) { controller.abort(); } return { requestId, controller }; @@ -72,13 +110,13 @@ export function registerRequestAbort( export function markRequestCanceled(requestId: string | undefined): void { if (!requestId) return; - evictOldestSetEntries(canceledRequestIds); - canceledRequestIds.add(requestId); + evictOldestSetEntries(canceledRequestIds()); + canceledRequestIds().add(requestId); // Abort WITH the typed error as the reason: every `signal.throwIfAborted()`, // fetch, and `throw signal.reason` downstream then surfaces the canceled // request as such, instead of a bare DOMException that normalizes to UNKNOWN. // No call site has to remember the factory — the signal carries it. - requestAbortControllers.get(requestId)?.abort(createRequestCanceledError()); + requestAbortControllers().get(requestId)?.abort(createRequestCanceledError()); } export function clearRequestCanceled( @@ -89,12 +127,12 @@ export function clearRequestCanceled( if ( registration && (registration.requestId !== requestId || - requestAbortControllers.get(requestId) !== registration.controller) + requestAbortControllers().get(requestId) !== registration.controller) ) { return; } - canceledRequestIds.delete(requestId); - requestAbortControllers.delete(requestId); + canceledRequestIds().delete(requestId); + requestAbortControllers().delete(requestId); } export function clearRequestAbortRegistration( @@ -106,12 +144,12 @@ export function clearRequestAbortRegistration( export function isRequestCanceled(requestId: string | undefined): boolean { if (!requestId) return false; - return canceledRequestIds.has(requestId); + return canceledRequestIds().has(requestId); } export function getRequestSignal(requestId: string | undefined): AbortSignal | undefined { if (!requestId) return undefined; - return requestAbortControllers.get(requestId)?.signal; + return requestAbortControllers().get(requestId)?.signal; } export function throwIfRequestCanceled(requestId: string | undefined): void { diff --git a/packages/host-kit/src/request.ts b/packages/host-kit/src/request.ts index ee98a01e7b..5ae44f36fc 100644 --- a/packages/host-kit/src/request.ts +++ b/packages/host-kit/src/request.ts @@ -1,5 +1,7 @@ export { clearRequestAbortRegistration, + createRequestCancelRuntimeState, + type RuntimeStateHandle, clearRequestCanceled, getRequestSignal, isRequestCanceled, From 9793a875085a7e6e809a45c2fd2dcd36c0488c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 23:30:02 +0200 Subject: [PATCH 03/11] feat(daemon): per-runtime Apple runner state and a shared PNG worker The Apple runner's sessions, locks, recycle ledger and owner state belong to the daemon runtime that composed them. The PNG worker is held by every runtime that uses it and stops with the last one. Co-Authored-By: Claude Sonnet 5.5 --- .../capture-kit/src/png-worker-client.test.ts | 17 +++++ packages/capture-kit/src/png-worker-client.ts | 18 ++++- .../support/daemon-runtime-deps-fixture.ts | 1 + .../daemon/src/core/interactor-resolution.ts | 6 +- .../src/core/provider-device-admission.ts | 6 +- .../daemon/src/resources/artifact-tracking.ts | 1 + .../daemon/src/server/http/daemon-runtime.ts | 22 ++++-- .../daemon/src/server/resumable-upload.ts | 1 + .../sessions/materialized-path-registry.ts | 1 + .../host-kit/src/internal/request-cancel.ts | 3 + .../platform-apple/src/core/runner-host.ts | 2 + .../src/core/runner-owner-state.test.ts | 40 +++++++++++ .../src/core/runner-owner-state.ts | 71 ++++++++++++++----- .../platform-apple/src/runner-owner-facade.ts | 1 + .../runner-session-close-finalization.test.ts | 2 +- .../runner-session-lifecycle.test.ts | 24 +++---- packages/platform-apple/src/runner/host.ts | 3 + .../src/runner/runner-recycle-ledger.ts | 26 +++---- .../src/runner/runner-session.ts | 57 ++++++++------- src/daemon-composition.ts | 4 ++ 20 files changed, 229 insertions(+), 77 deletions(-) create mode 100644 packages/platform-apple/src/core/runner-owner-state.test.ts diff --git a/packages/capture-kit/src/png-worker-client.test.ts b/packages/capture-kit/src/png-worker-client.test.ts index 8c9835b713..847fd5126f 100644 --- a/packages/capture-kit/src/png-worker-client.test.ts +++ b/packages/capture-kit/src/png-worker-client.test.ts @@ -9,6 +9,7 @@ import { decodePngAsync, decodeScreenshotImageAsync, encodePngAsync, + retainPngWorker, terminatePngWorker, transcodeScreenshotToPngAsync, } from './png-worker-client.ts'; @@ -225,3 +226,19 @@ test('decodeScreenshotImageAsync rejects a container it cannot read with the can }, ); }); + +test('the worker serves every holder and stops only when the last one lets go', async () => { + const first = retainPngWorker(); + const second = retainPngWorker(); + const encoded = PNG.sync.write(buildPatternPng(5, 4, 2)); + + await first(); + await first(); + const stillServing = await runOnWorkerThread('decode after one release', () => + decodePngAsync(encoded, 'fixture'), + ); + assert.equal(stillServing.width, 5); + + await second(); + assert.equal(process.getActiveResourcesInfo().includes('MessagePort'), false); +}); diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index 994e1cedf0..e66ad5aba4 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -181,9 +181,23 @@ async function runPngJob( } } -/** Daemon startup hook: spawns the worker before the first screenshot job. */ -export function prewarmPngWorker(): void { +let pngWorkerHolds = 0; + +/** + * Daemon startup hook: spawns the worker before the first screenshot job and holds it for the + * caller. The worker is shared by every runtime in the process, so it stops when the last holder + * lets go; the returned release is idempotent. + */ +export function retainPngWorker(): () => Promise { obtainWorker(); + pngWorkerHolds += 1; + let released = false; + return async () => { + if (released) return; + released = true; + pngWorkerHolds -= 1; + if (pngWorkerHolds === 0) await terminatePngWorker(); + }; } /** Stops the worker thread (used by tests and shutdown); later calls respawn it. */ diff --git a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts index ea50b0b8e7..0e7cbc42ff 100644 --- a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts +++ b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts @@ -59,6 +59,7 @@ export function daemonRuntimeDepsFixture( return async (input) => { const resolved = typeof overrides === 'function' ? overrides(input) : overrides; return Object.freeze({ + runtimeStates: [], ownedProcessRecords: createOwnedProcessRecordStore({ stateDir: input.baseDir, sessionsDir: input.sessionsDir, diff --git a/packages/daemon/src/core/interactor-resolution.ts b/packages/daemon/src/core/interactor-resolution.ts index aea563e018..61d68ac2c6 100644 --- a/packages/daemon/src/core/interactor-resolution.ts +++ b/packages/daemon/src/core/interactor-resolution.ts @@ -41,7 +41,11 @@ const interactorResolutionState = (): InteractorResolutionState => /** The resolution one daemon runtime owns; the runtime installs its composition's into it. */ export function createInteractorResolutionRuntimeState(): RuntimeStateHandle { const owned: InteractorResolutionState = { resolution: NO_INTERACTOR_RESOLUTION }; - return { run: (fn) => interactorResolutionStorage.run(owned, fn), dispose: () => {} }; + return { + run: (fn) => interactorResolutionStorage.run(owned, fn), + enter: () => interactorResolutionStorage.enterWith(owned), + dispose: () => {}, + }; } /** diff --git a/packages/daemon/src/core/provider-device-admission.ts b/packages/daemon/src/core/provider-device-admission.ts index 8185f6dc9a..eadf0de88d 100644 --- a/packages/daemon/src/core/provider-device-admission.ts +++ b/packages/daemon/src/core/provider-device-admission.ts @@ -32,7 +32,11 @@ const providerAdmissionState = (): ProviderAdmissionState => /** The admission one daemon runtime owns; the runtime installs its composition's into it. */ export function createProviderDeviceAdmissionRuntimeState(): RuntimeStateHandle { const owned: ProviderAdmissionState = { admission: NO_PROVIDER_DEVICE_ADMISSION }; - return { run: (fn) => providerAdmissionStorage.run(owned, fn), dispose: () => {} }; + return { + run: (fn) => providerAdmissionStorage.run(owned, fn), + enter: () => providerAdmissionStorage.enterWith(owned), + dispose: () => {}, + }; } /** diff --git a/packages/daemon/src/resources/artifact-tracking.ts b/packages/daemon/src/resources/artifact-tracking.ts index 2fcb74fdf3..124c7321c0 100644 --- a/packages/daemon/src/resources/artifact-tracking.ts +++ b/packages/daemon/src/resources/artifact-tracking.ts @@ -67,6 +67,7 @@ export function createArtifactTrackingRuntimeState(): RuntimeStateHandle { const owned = createArtifactTrackingState(); return { run: (fn) => artifactTrackingStorage.run(owned, fn), + enter: () => artifactTrackingStorage.enterWith(owned), dispose: () => { for (const entry of [...owned.artifacts.values(), ...owned.uploads.values()]) { clearTimeout(entry.timer); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index f3795bb549..ce4990409e 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -102,7 +102,7 @@ import { listenNetServer, type DaemonServer, } from './transport.ts'; -import { prewarmPngWorker, terminatePngWorker } from '@agent-device/capture-kit/png-worker-client'; +import { retainPngWorker } from '@agent-device/capture-kit/png-worker-client'; import { openWebSessionNames } from '../../sessions/web-session-names.ts'; import { createAppLogAdmissionLedger } from '../../resources/app-log-admission-ledger.ts'; import type { AppLogSessionArtifacts } from '@agent-device/contracts/app-log-runtime'; @@ -138,6 +138,8 @@ type WritableOutput = { * same factory, so the production assembly is the one every runtime test boots. */ export type DaemonRuntimeDeps = Readonly<{ + /** State the composition's platform modules keep per daemon runtime (the Apple runner's). */ + runtimeStates: readonly RuntimeStateHandle[]; ownedProcessRecords: OwnedProcessRecordStore; hostDiagnostics: HostDiagnostics; ownerLifecycle: PlatformOwnerLifecycle; @@ -425,7 +427,12 @@ export async function teardownDaemonSessionForShutdown(params: { return teardownSucceeded; } -type RuntimeContext = Readonly<{ run(fn: () => T): T; dispose(): void }>; +type RuntimeContext = Readonly<{ + run(fn: () => T): T; + /** Adds state the composition contributed and binds it for the rest of the start. */ + extend(handles: readonly RuntimeStateHandle[]): void; + dispose(): void; +}>; /** The per-runtime state of every module that keeps some, in the order it is entered. */ function createDaemonRuntimeStates(): RuntimeStateHandle[] { @@ -445,6 +452,12 @@ export async function startDaemonRuntime( const states = createDaemonRuntimeStates(); const runtimeContext: RuntimeContext = { run: (fn) => states.reduceRight((next, state) => () => state.run(next), fn)(), + extend: (handles) => { + for (const handle of handles) { + states.push(handle); + handle.enter(); + } + }, dispose: () => { for (const state of states) state.dispose(); }, @@ -500,6 +513,7 @@ async function startDaemonRuntimeOnHost( }), openWebSessionNames: () => openWebSessionNames(sessionStore), }); + runtimeContext.extend(deps.runtimeStates); const appLogAdmissionLedger = createAppLogAdmissionLedger(); const audioProbeAdmissionLedger = createAudioProbeAdmissionLedger(); const perfCaptureAdmissionLedger = createPerfCaptureAdmissionLedger(); @@ -917,7 +931,7 @@ async function startDaemonRuntimeOnHost( // Spawn the PNG worker ahead of the first screenshot request so its // cold-start cost is not paid on a user-visible call. Best effort: when it // cannot start, PNG processing falls back to the in-process sync path. - prewarmPngWorker(); + const releasePngWorker = retainPngWorker(); const closeOnHost = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); @@ -982,7 +996,7 @@ async function startDaemonRuntimeOnHost( const stoppedRunners = await applicationLifecycle.finalizeDaemonShutdown(); // Best effort: stop the PNG worker so an in-flight job cannot delay exit. await Promise.race([ - terminatePngWorker().catch(() => {}), + releasePngWorker().catch(() => {}), sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), ]); await finishDaemonRegistration({ diff --git a/packages/daemon/src/server/resumable-upload.ts b/packages/daemon/src/server/resumable-upload.ts index fd6a8f9baa..e39d1bd28b 100644 --- a/packages/daemon/src/server/resumable-upload.ts +++ b/packages/daemon/src/server/resumable-upload.ts @@ -43,6 +43,7 @@ export function createResumableUploadRuntimeState(): RuntimeStateHandle { const owned = createResumableUploadState(); return { run: (fn) => resumableUploadStorage.run(owned, fn), + enter: () => resumableUploadStorage.enterWith(owned), dispose: () => { for (const entry of owned.byId.values()) clearTimeout(entry.timer); owned.byId.clear(); diff --git a/packages/daemon/src/sessions/materialized-path-registry.ts b/packages/daemon/src/sessions/materialized-path-registry.ts index 57494f7481..b2d7f15137 100644 --- a/packages/daemon/src/sessions/materialized-path-registry.ts +++ b/packages/daemon/src/sessions/materialized-path-registry.ts @@ -45,6 +45,7 @@ export function createMaterializedPathRuntimeState(): RuntimeStateHandle { const owned = new Map(); return { run: (fn) => retainedPathsStorage.run(owned, fn), + enter: () => retainedPathsStorage.enterWith(owned), dispose: () => { for (const entry of owned.values()) clearTimeout(entry.timer); owned.clear(); diff --git a/packages/host-kit/src/internal/request-cancel.ts b/packages/host-kit/src/internal/request-cancel.ts index 091cfa326a..09a3122ea0 100644 --- a/packages/host-kit/src/internal/request-cancel.ts +++ b/packages/host-kit/src/internal/request-cancel.ts @@ -9,6 +9,8 @@ import { AppError, createRequestCanceledError } from '@agent-device/kernel/error export type RuntimeStateHandle = Readonly<{ /** Runs `fn` as the runtime that owns this state. */ run(fn: () => T): T; + /** Binds this state for the rest of the current async context, after the runtime is composed. */ + enter(): void; /** Releases what the state holds (timers, handles). */ dispose(): void; }>; @@ -33,6 +35,7 @@ export function createRequestCancelRuntimeState(): RuntimeStateHandle { const owned = createRequestCancelState(); return { run: (fn) => requestCancelStorage.run(owned, fn), + enter: () => requestCancelStorage.enterWith(owned), dispose: () => { for (const controller of owned.requestAbortControllers.values()) controller.abort(); owned.requestAbortControllers.clear(); diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index 6790d89a7c..f84ea4b4a1 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -42,6 +42,7 @@ import { visitXmlPlistEntries } from './plist-xml.ts'; import { getRunnerDeviceClaimAuthorityProbe, getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, } from './runner-owner-state.ts'; import { buildSimctlArgsForDevice, simulatorAddressFor } from './simctl.ts'; import { @@ -108,5 +109,6 @@ export const appleRunnerHost: AppleRunnerHost = { resolveIosPhysicalDeviceControl, visitXmlPlistEntries, leaseOwnerStateDir: getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, hasDeviceClaimAuthority: (device) => getRunnerDeviceClaimAuthorityProbe()?.(device) ?? false, }; diff --git a/packages/platform-apple/src/core/runner-owner-state.test.ts b/packages/platform-apple/src/core/runner-owner-state.test.ts new file mode 100644 index 0000000000..fa4de52b7a --- /dev/null +++ b/packages/platform-apple/src/core/runner-owner-state.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from 'vitest'; +import { + createAppleRunnerRuntimeState, + getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, + setRunnerLeaseOwnerStateDir, +} from './runner-owner-state.ts'; + +const SLOT = Symbol('test-slot'); + +test('each runtime keeps its own owner state and registries', () => { + const first = createAppleRunnerRuntimeState(); + const second = createAppleRunnerRuntimeState(); + + first.run(() => { + setRunnerLeaseOwnerStateDir(' /state/first '); + runnerRuntimeSlot(SLOT, () => new Map()).set('device', 'first'); + }); + second.run(() => { + setRunnerLeaseOwnerStateDir('/state/second'); + }); + + expect(first.run(() => getRunnerLeaseOwnerStateDir())).toBe('/state/first'); + expect(second.run(() => getRunnerLeaseOwnerStateDir())).toBe('/state/second'); + expect( + first.run(() => runnerRuntimeSlot(SLOT, () => new Map()).get('device')), + ).toBe('first'); + expect(second.run(() => runnerRuntimeSlot(SLOT, () => new Map()).size)).toBe(0); + // Nothing leaked into the standalone state the code outside any runtime reads. + expect(getRunnerLeaseOwnerStateDir()).toBeUndefined(); +}); + +test('disposing a runtime drops its registries', () => { + const runtime = createAppleRunnerRuntimeState(); + runtime.run(() => runnerRuntimeSlot(SLOT, () => new Map([['device', 'owned']]))); + + runtime.dispose(); + + expect(runtime.run(() => runnerRuntimeSlot(SLOT, () => new Map()).size)).toBe(0); +}); diff --git a/packages/platform-apple/src/core/runner-owner-state.ts b/packages/platform-apple/src/core/runner-owner-state.ts index e44dddd94e..534f58e3ca 100644 --- a/packages/platform-apple/src/core/runner-owner-state.ts +++ b/packages/platform-apple/src/core/runner-owner-state.ts @@ -1,20 +1,6 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; import type { DeviceInfo } from '@agent-device/kernel/device'; - -/** - * Daemon-owned lease-owner state directory for the Apple runner. Kept in a - * dedicated tiny module so daemon startup can set it without loading the - * runner client's implementation closure; the runner package reads it through - * its host port at lease time. - */ -let runnerLeaseOwnerStateDir: string | undefined; - -export function setRunnerLeaseOwnerStateDir(stateDir: string | undefined): void { - runnerLeaseOwnerStateDir = stateDir?.trim() || undefined; -} - -export function getRunnerLeaseOwnerStateDir(): string | undefined { - return runnerLeaseOwnerStateDir; -} +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; /** * Daemon-owned device-claim arbitration probe: does the embedding process hold @@ -26,14 +12,61 @@ export function getRunnerLeaseOwnerStateDir(): string | undefined { */ export type RunnerDeviceClaimAuthorityProbe = (device: DeviceInfo) => boolean; -let runnerDeviceClaimAuthorityProbe: RunnerDeviceClaimAuthorityProbe | undefined; +/** + * What one daemon runtime owns of the Apple runner: the lease-owner state directory and claim probe + * its daemon set, and the registries (sessions, locks, recycle ledger) the runner modules keep in + * {@link runnerRuntimeSlot}s. Kept in a dedicated tiny module so daemon startup can reach it + * without loading the runner client's implementation closure; the runner package reads it through + * its host port. + */ +type RunnerRuntimeState = { + leaseOwnerStateDir: string | undefined; + deviceClaimAuthorityProbe: RunnerDeviceClaimAuthorityProbe | undefined; + slots: Map; +}; + +const createRunnerRuntimeState = (): RunnerRuntimeState => ({ + leaseOwnerStateDir: undefined, + deviceClaimAuthorityProbe: undefined, + slots: new Map(), +}); + +const runnerRuntimeStorage = new AsyncLocalStorage(); +// Outside a daemon runtime (a package test, a tool that drives the runner directly) this answers. +const standaloneRunnerRuntime = createRunnerRuntimeState(); +const runnerRuntimeState = (): RunnerRuntimeState => + runnerRuntimeStorage.getStore() ?? standaloneRunnerRuntime; + +export function createAppleRunnerRuntimeState(): RuntimeStateHandle { + const owned = createRunnerRuntimeState(); + return { + run: (fn) => runnerRuntimeStorage.run(owned, fn), + enter: () => runnerRuntimeStorage.enterWith(owned), + dispose: () => owned.slots.clear(), + }; +} + +/** One piece of runner state this daemon runtime owns, created on first use. */ +export function runnerRuntimeSlot(key: symbol, create: () => T): T { + const { slots } = runnerRuntimeState(); + if (!slots.has(key)) slots.set(key, create()); + return slots.get(key) as T; +} + +export function setRunnerLeaseOwnerStateDir(stateDir: string | undefined): void { + runnerRuntimeState().leaseOwnerStateDir = stateDir?.trim() || undefined; +} + +export function getRunnerLeaseOwnerStateDir(): string | undefined { + return runnerRuntimeState().leaseOwnerStateDir; +} export function setRunnerDeviceClaimAuthorityProbe( probe: RunnerDeviceClaimAuthorityProbe | undefined, ): void { - runnerDeviceClaimAuthorityProbe = probe; + runnerRuntimeState().deviceClaimAuthorityProbe = probe; } export function getRunnerDeviceClaimAuthorityProbe(): RunnerDeviceClaimAuthorityProbe | undefined { - return runnerDeviceClaimAuthorityProbe; + return runnerRuntimeState().deviceClaimAuthorityProbe; } diff --git a/packages/platform-apple/src/runner-owner-facade.ts b/packages/platform-apple/src/runner-owner-facade.ts index 9b683958d8..123e115748 100644 --- a/packages/platform-apple/src/runner-owner-facade.ts +++ b/packages/platform-apple/src/runner-owner-facade.ts @@ -1,4 +1,5 @@ export { + createAppleRunnerRuntimeState, setRunnerDeviceClaimAuthorityProbe, setRunnerLeaseOwnerStateDir, type RunnerDeviceClaimAuthorityProbe, diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts index a5ecc5270f..e8436b5f97 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts @@ -381,7 +381,7 @@ test('graceful shutdown stops idle retained generations and hands off active rea const active = await ensureReadyRunner(activeDevice); await closeFinalizationLifecycle(idleDevice, []).finalizeApplicationClose(closeInput(true)); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(active.state, 'stopped'); assert.equal(idle.state, 'ready'); await stopAllIosRunnerSessions(); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts index ab7a494392..2251d23ec9 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts @@ -343,7 +343,7 @@ test('shutdown detach moves a handed-off session to stopped without killing its assert.ok(runnerPid); runnerStateTransitions.length = 0; - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(session.state, 'stopped'); assert.deepEqual(runnerStateTransitions, ['stopped']); @@ -361,7 +361,7 @@ test('a scoped simulator-set session hands off like one in the default set', asy const session = await ensureRunnerSession(device, {}); await serveOneCommand(device, session); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(session.state, 'stopped'); assert.match(leaseRaw(device.id), /"ownerToken": "detached-owner-/); @@ -393,7 +393,7 @@ test('a runner that served a command is handed off on the physical lane', async runnerStateTransitions.length = 0; const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); }); assert.equal(session.state, 'stopped'); @@ -418,7 +418,7 @@ test('a runner that never served a command is torn down instead of handed off', assert.equal(session.state, 'starting'); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.deepEqual(readRunnerSessionLiveness(device.id), { @@ -449,7 +449,7 @@ test('a runner reporting main-thread work still draining is not handed off', asy assert.equal(session.state, 'ready'); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"main_thread_occupied"/); @@ -484,7 +484,7 @@ test('a session that still owes a response is not handed off', async () => { assert.equal(session.commandCharges.outstandingChargeCount, 1); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"command_in_flight"/); @@ -521,7 +521,7 @@ test('a command abandoned by a cancelled transport keeps the runner occupied', a assert.equal(session.commandCharges.hasAbandonedCharges, true); const refused = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(refused, /"reason":"command_in_flight"/); // The refusal this issue is about: nothing is awaited any more, and only terminal evidence for that @@ -535,7 +535,7 @@ test('a command abandoned by a cancelled transport keeps the runner occupied', a // `runner-client-recovery-wiring.test.ts`, where the probe reaches a real session. await serveOneCommand(device, session); assert.equal(session.commandCharges.hasOutstandingCharges, false); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); }); test('the macOS host runner is never handed off, although it is kind device', async () => { @@ -544,7 +544,7 @@ test('the macOS host runner is never handed off, although it is kind device', as await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"macos_host"/); @@ -563,7 +563,7 @@ test('a physical tvOS runner is never handed off, although it is kind device too await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"physical_non_ios_os"/); @@ -580,7 +580,7 @@ test('a usbmux-only xctest backend runner is never handed off', async () => { await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"xctest_backend"/); @@ -597,7 +597,7 @@ test("the handoff releases this daemon's log bookkeeping only after the lease sa tokenWhenReleased.push(leaseRaw(device.id)); }; - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(tokenWhenReleased.length, 1); assert.match(tokenWhenReleased[0]!, /"ownerToken": "detached-owner-/); diff --git a/packages/platform-apple/src/runner/host.ts b/packages/platform-apple/src/runner/host.ts index 37be783617..dbc631f8bc 100644 --- a/packages/platform-apple/src/runner/host.ts +++ b/packages/platform-apple/src/runner/host.ts @@ -97,6 +97,8 @@ export type AppleRunnerHost = Pick< resolveIosPhysicalDeviceControl: (device: DeviceInfo) => IosPhysicalDeviceRunnerControl; /** Daemon-owned lease owner state directory. */ leaseOwnerStateDir: typeof AppleRunnerOwnerState.getRunnerLeaseOwnerStateDir; + /** One registry the daemon runtime owns (sessions, locks, ledgers), created on first use. */ + runnerRuntimeSlot: typeof AppleRunnerOwnerState.runnerRuntimeSlot; /** * Daemon-owned device-claim arbitration probe: true only while the embedding process holds * the host-global local device claim for exactly this device (matched by canonical @@ -173,6 +175,7 @@ export const classifyOwnerLiveness = delegate('classifyOwnerLiveness'); export const readHostCpuArchSync = delegate('readHostCpuArchSync'); export const findProjectRoot = delegate('findProjectRoot'); export const readVersion = delegate('readVersion'); +export const runnerRuntimeSlot = delegate('runnerRuntimeSlot'); export const acquireProcessLock = delegate('acquireProcessLock'); export const withProcessLock = delegate('withProcessLock'); export const publishFileSync = delegate('publishFileSync'); diff --git a/packages/platform-apple/src/runner/runner-recycle-ledger.ts b/packages/platform-apple/src/runner/runner-recycle-ledger.ts index 149e681ea3..8af8fbd867 100644 --- a/packages/platform-apple/src/runner/runner-recycle-ledger.ts +++ b/packages/platform-apple/src/runner/runner-recycle-ledger.ts @@ -1,5 +1,5 @@ import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; -import { emitDiagnostic } from './host.ts'; +import { emitDiagnostic, runnerRuntimeSlot } from './host.ts'; import type { RunnerCommand } from './runner-contract.ts'; // Caps how many times one daemon request may pay for a full runner recycle @@ -19,7 +19,9 @@ type RunnerRecycleLedgerEntry = { lastAtMs: number; }; -const ledger = new Map(); +const LEDGER_SLOT = Symbol('apple-runner-recycle-ledger'); +const ledgerEntries = (): Map => + runnerRuntimeSlot(LEDGER_SLOT, () => new Map()); /** Requests are the recycle scope; direct calls without a request fall back to the command id. */ export function runnerRecycleLedgerKey( @@ -43,7 +45,7 @@ export function markRunnerRequestTouchedSession(key: string | undefined): void { export function hasRunnerRequestTouchedSession(key: string | undefined): boolean { if (!key) return false; - return ledger.get(key)?.touchedSession === true; + return ledgerEntries().get(key)?.touchedSession === true; } /** @@ -108,28 +110,28 @@ export function buildRunnerRecycleBudgetExhaustedError( * @internal Test isolation hook for process-global runner recycle accounting. */ export function resetRunnerRecycleLedgerForTests(): void { - ledger.clear(); + ledgerEntries().clear(); } function readEntry(key: string): RunnerRecycleLedgerEntry { pruneLedger(); - return ledger.get(key) ?? { touchedSession: false, recycles: 0, lastAtMs: Date.now() }; + return ledgerEntries().get(key) ?? { touchedSession: false, recycles: 0, lastAtMs: Date.now() }; } function writeEntry(key: string, entry: RunnerRecycleLedgerEntry): void { entry.lastAtMs = Date.now(); - ledger.delete(key); - ledger.set(key, entry); + ledgerEntries().delete(key); + ledgerEntries().set(key, entry); } function pruneLedger(): void { const cutoff = Date.now() - LEDGER_ENTRY_TTL_MS; - for (const [key, entry] of ledger) { - if (entry.lastAtMs < cutoff) ledger.delete(key); + for (const [key, entry] of ledgerEntries()) { + if (entry.lastAtMs < cutoff) ledgerEntries().delete(key); } - while (ledger.size > LEDGER_MAX_ENTRIES) { - const oldest = ledger.keys().next().value; + while (ledgerEntries().size > LEDGER_MAX_ENTRIES) { + const oldest = ledgerEntries().keys().next().value; if (oldest === undefined) break; - ledger.delete(oldest); + ledgerEntries().delete(oldest); } } diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 87b8bfb405..243436a06f 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -1,5 +1,6 @@ import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { + runnerRuntimeSlot, withKeyedLock, Deadline, emitRequestProgress, @@ -84,12 +85,17 @@ export type { RunnerSession } from './runner-session-types.ts'; export type RunnerSessionOptions = AppleRunnerLifecycleOptions; -const runnerSessions = new Map(); -const runnerSessionLocks = new Map>(); +const RUNNER_SESSIONS_SLOT = Symbol('apple-runner-sessions'); +const RUNNER_SESSION_LOCKS_SLOT = Symbol('apple-runner-session-locks'); +// The daemon runtime's registries: two runtimes in one process never see each other's runners. +const runnerSessions = () => + runnerRuntimeSlot(RUNNER_SESSIONS_SLOT, () => new Map()); +const runnerSessionLocks = () => + runnerRuntimeSlot(RUNNER_SESSION_LOCKS_SLOT, () => new Map>()); const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; function withRunnerSessionLock(deviceId: string, task: () => Promise): Promise { - return withKeyedLock(runnerSessionLocks, deviceId, task); + return withKeyedLock(runnerSessionLocks(), deviceId, task); } export async function ensureRunnerSession( @@ -122,7 +128,7 @@ export async function ensureRunnerSession( // a joiner's cancellation can never outvote the live owner and stop its build (#3220). const releaseOwnerInterest = reserveRunnerStartOwnerInterest(startAdmission, options); try { - const existing = runnerSessions.get(device.id); + const existing = runnerSessions().get(device.id); if (existing) { assertExpectedRunnerSession(existing, options.expectedRunnerSessionId); const reusable = await resolveReusableRunnerSession(device, existing, budget.phase); @@ -195,7 +201,7 @@ async function startRunnerSessionWithLease( // gate: a start fenced by a teardown registers its runner into no device (#3220). The // runner itself keeps its lease for the next open to adopt. assertRunnerStartAdmitsPreparation(device.id, startAdmission); - runnerSessions.set(device.id, adopted); + runnerSessions().set(device.id, adopted); return adopted; } assertRunnerSessionMayStart(options.expectedRunnerSessionId); @@ -369,7 +375,7 @@ async function startRunnerSessionWithLease( }); throw error; } - runnerSessions.set(device.id, session); + runnerSessions().set(device.id, session); return session; } @@ -560,7 +566,7 @@ async function uninstallStaleSimulatorRunnerBundle( * registered, which is its own answer: there is no session to wait for or tear down. */ export function readRunnerSessionLiveness(deviceId: string): RunnerSessionRegistration | null { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session) return null; return { sessionId: session.sessionId, @@ -580,7 +586,7 @@ export async function invalidateRunnerSession( reason: string, ): Promise { await withRunnerSessionLock(session.deviceId, async () => { - if (runnerSessions.get(session.deviceId) !== session) return; + if (runnerSessions().get(session.deviceId) !== session) return; // A session already being torn down, or already torn down, is never disposed a second time // for a later reason; the reason-coded diagnostic below reports why this call was made. if (!canWorkWithRunnerSession(session)) return; @@ -605,15 +611,15 @@ async function stopRunnerSessionInternal( sessionOverride?: RunnerSession, options: RunnerDisposalOptions = {}, ): Promise { - const session = sessionOverride ?? runnerSessions.get(deviceId); + const session = sessionOverride ?? runnerSessions().get(deviceId); if (!session) return; // Once disposal has begun or finished, this session has no runner to wait on; a repeat stop // would only re-signal a process that is already leaving and re-emit a teardown for a reason // that has nothing left to tear down. if (!canWorkWithRunnerSession(session)) return; await disposeRunnerSession(session, options); - if (runnerSessions.get(deviceId) === session) { - runnerSessions.delete(deviceId); + if (runnerSessions().get(deviceId) === session) { + runnerSessions().delete(deviceId); } } @@ -623,7 +629,8 @@ async function stopRetainedRunner( reason: 'idle_timeout' | 'listener_lost', ): Promise { await withRunnerSessionLock(session.deviceId, async () => { - if (runnerSessions.get(session.deviceId) !== session || session.retention !== retention) return; + if (runnerSessions().get(session.deviceId) !== session || session.retention !== retention) + return; emitDiagnostic({ level: reason === 'listener_lost' ? 'warn' : 'info', phase: reason === 'listener_lost' ? 'ios_runner_warm_stop' : 'ios_runner_idle_stop', @@ -652,7 +659,7 @@ export async function releaseSpeculativeIosRunnerSession(deviceId: string): Prom // Under the session lock: a prewarm still starting holds it and registers its session only // when the start completes, so the release queues behind that start instead of missing it. return await withRunnerSessionLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session?.speculative) return false; emitDiagnostic({ level: 'debug', @@ -684,7 +691,7 @@ async function stopIosRunnerSessionDevice( try { await withRunnerSessionLock(deviceId, async () => { await withRunnerLeaseLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); await stopRunnerSessionInternal(deviceId, session, { leaseLockHeld: true, graceful: @@ -706,13 +713,13 @@ export async function releaseIosRunnerOnClose( deviceId: string, options: { retain: boolean }, ): Promise { - const candidate = runnerSessions.get(deviceId); + const candidate = runnerSessions().get(deviceId); const busyAtClose = isRunnerMainThreadOccupied(candidate); const settleFence = fenceRunnerStartAdmissionsForTeardown(deviceId); try { if (options.retain && candidate?.state === 'ready' && !busyAtClose) { const retained = await withRunnerSessionLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session) return false; const { retainRunnerSession } = await import('./runner-retention.ts'); if ( @@ -751,15 +758,15 @@ export async function abortAllIosRunnerSessions(): Promise { // The same fence a close raises, for every device at once: the prep sweep the abort performs is // one-shot, so an in-flight start that would answer it with another build is refused before the // sweep runs, and stays refused until the sweep settles (#3220). - const activeSessions = Array.from(runnerSessions.values()); + const activeSessions = Array.from(runnerSessions().values()); const settleFences = retireAllRunnerStartAdmissions( activeSessions.map((session) => session.deviceId), ); try { await abortRunnerSessionsAndPrepProcesses(activeSessions); for (const session of activeSessions) { - if (runnerSessions.get(session.deviceId) === session) { - runnerSessions.delete(session.deviceId); + if (runnerSessions().get(session.deviceId) === session) { + runnerSessions().delete(session.deviceId); } } } finally { @@ -772,21 +779,21 @@ export async function abortAllIosRunnerSessions(): Promise { // (#2681); detached active generations leave the registry. Idle retained generations stay // owned until shutdown stops them. export async function detachIosRunnerSessionsForShutdown(): Promise { - return await detachRunnerSessionsForShutdown(runnerSessions, (deviceId) => { - runnerSessions.delete(deviceId); + return await detachRunnerSessionsForShutdown(runnerSessions(), (deviceId) => { + runnerSessions().delete(deviceId); }); } /** Stops every remaining runner and names the devices it owned. */ export async function stopAllIosRunnerSessions(): Promise { - const owned = [...runnerSessions.keys()]; + const owned = [...runnerSessions().keys()]; // This is the daemon's teardown: the fence spans the whole sweep, not just the abort, so a // start that began while the per-session stops ran meets a closed admission at its next gate // instead of answering the final prep sweep with another build (#3220). - const settleFences = retireAllRunnerStartAdmissions(runnerSessions.keys()); + const settleFences = retireAllRunnerStartAdmissions(runnerSessions().keys()); try { await abortAllIosRunnerSessions(); - const pending = Array.from(runnerSessions.keys()); + const pending = Array.from(runnerSessions().keys()); await Promise.allSettled( pending.map(async (deviceId) => { await stopIosRunnerSession(deviceId); @@ -855,7 +862,7 @@ export async function executeRunnerCommandWithSession( (reason) => invalidateRunnerSession(session, reason), signal, ); - if (runnerSessions.get(device.id) === session && session.state === 'ready') { + if (runnerSessions().get(device.id) === session && session.state === 'ready') { const { observeRunnerListener } = await import('./runner-retention.ts'); observeRunnerListener(session, () => { const retention = session.retention; diff --git a/src/daemon-composition.ts b/src/daemon-composition.ts index 773763d24b..ec736f40b5 100644 --- a/src/daemon-composition.ts +++ b/src/daemon-composition.ts @@ -125,7 +125,11 @@ export const createDaemonRuntimeDeps: ComposeDaemonRuntimeDeps = async ( const { createDaemonPlatformServices } = await import('./platform-runtime-daemon-services.ts'); const { createClaimRecoveryGateway } = await import('./platform-runtime-claim-recovery.ts'); + const { createAppleRunnerRuntimeState } = + await import('@agent-device/platform-apple/runner-owner'); + return Object.freeze({ + runtimeStates: Object.freeze([createAppleRunnerRuntimeState()]), ownedProcessRecords, hostDiagnostics, ownerLifecycle: platformDaemonLifecycleOwners, From 0b83711eb15718a627740ea56c5a9c26bc62c0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 01:50:23 +0200 Subject: [PATCH 04/11] chore(daemon): drop re-exports nothing consumes yet Co-Authored-By: Claude Sonnet 5.5 --- packages/daemon/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index b3204cc2cd..a72685e11c 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -1,10 +1,8 @@ export { daemonStartupExitCode, startDaemonRuntime } from './server/http/daemon-runtime.ts'; -export type { ShutdownReport } from './server/http/server-shutdown.ts'; export type { ComposeDaemonRuntimeDeps, DaemonRuntimeController, DaemonRuntimeDeps, - DaemonRuntimeDepsInput, } from './server/http/daemon-runtime.ts'; export type { DaemonInvokeFn, DaemonRequest } from './core/daemon-request.ts'; export type { DaemonPlatformServices } from './core/platform-services.ts'; From c3baaeccc910f344474756e3c52ed26363380e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 02:17:35 +0200 Subject: [PATCH 05/11] fix(daemon): a failed shutdown step releases the rest, and dispose removes temp files Co-Authored-By: Claude Sonnet 5.5 --- .../src/resources/artifact-tracking.test.ts | 32 +++++++++ .../daemon/src/resources/artifact-tracking.ts | 16 ++++- .../server/http/daemon-runtime-close.test.ts | 72 +++++++++++++++---- .../http/daemon-runtime-composition.test.ts | 7 +- .../http/daemon-runtime-isolation.test.ts | 10 +-- .../daemon/src/server/http/daemon-runtime.ts | 25 ++++++- .../daemon/src/server/resumable-upload.ts | 9 ++- .../sessions/materialized-path-registry.ts | 10 ++- src/daemon.ts | 8 +-- 9 files changed, 158 insertions(+), 31 deletions(-) create mode 100644 packages/daemon/src/resources/artifact-tracking.test.ts diff --git a/packages/daemon/src/resources/artifact-tracking.test.ts b/packages/daemon/src/resources/artifact-tracking.test.ts new file mode 100644 index 0000000000..c2e4538847 --- /dev/null +++ b/packages/daemon/src/resources/artifact-tracking.test.ts @@ -0,0 +1,32 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { + createArtifactTrackingRuntimeState, + trackDownloadableArtifact, + trackUploadedArtifact, +} from './artifact-tracking.ts'; + +test('disposing a runtime removes the temporary files it still tracks', () => { + const root = mkdtempForTestSync('agent-device-artifact-dispose-'); + const artifactPath = path.join(root, 'artifact.txt'); + const uploadDir = path.join(root, 'upload'); + fs.writeFileSync(artifactPath, 'artifact'); + fs.mkdirSync(uploadDir); + fs.writeFileSync(path.join(uploadDir, 'payload'), 'payload'); + const runtime = createArtifactTrackingRuntimeState(); + + runtime.run(() => { + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'artifact.txt', + }); + trackUploadedArtifact({ artifactPath: path.join(uploadDir, 'payload'), tempDir: uploadDir }); + }); + runtime.dispose(); + + expect(fs.existsSync(uploadDir)).toBe(false); + expect(fs.existsSync(artifactPath)).toBe(false); +}); diff --git a/packages/daemon/src/resources/artifact-tracking.ts b/packages/daemon/src/resources/artifact-tracking.ts index 124c7321c0..f0c827bcf1 100644 --- a/packages/daemon/src/resources/artifact-tracking.ts +++ b/packages/daemon/src/resources/artifact-tracking.ts @@ -69,8 +69,14 @@ export function createArtifactTrackingRuntimeState(): RuntimeStateHandle { run: (fn) => artifactTrackingStorage.run(owned, fn), enter: () => artifactTrackingStorage.enterWith(owned), dispose: () => { - for (const entry of [...owned.artifacts.values(), ...owned.uploads.values()]) { + for (const entry of owned.artifacts.values()) { clearTimeout(entry.timer); + cleanupDirectoryArchive(entry); + if (entry.deleteAfterDownload) removeBestEffort(entry.artifactPath); + } + for (const entry of owned.uploads.values()) { + clearTimeout(entry.timer); + removeBestEffort(entry.tempDir); } owned.artifacts.clear(); owned.uploads.clear(); @@ -339,6 +345,14 @@ export function cleanupDownloadableArtifact(artifactId: string): void { } } +function removeBestEffort(targetPath: string): void { + try { + fs.rmSync(targetPath, { recursive: true, force: true }); + } catch { + // best-effort cleanup only + } +} + function cleanupDirectoryArchive(entry: ArtifactEntry): void { if (!entry.directoryArchive) return; try { diff --git a/packages/daemon/src/server/http/daemon-runtime-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-close.test.ts index 480f80df5e..0b30835d26 100644 --- a/packages/daemon/src/server/http/daemon-runtime-close.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-close.test.ts @@ -1,4 +1,6 @@ +import fs from 'node:fs'; import { expect, test, vi } from 'vitest'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; import { @@ -47,23 +49,33 @@ test('close is idempotent and every caller receives the one report', async () => }), }), }); - expect(process.listenerCount('SIGTERM')).toBe(signalListeners); + try { + expect(process.listenerCount('SIGTERM')).toBe(signalListeners); - const [first, second] = await Promise.all([runtime.close(), runtime.close()]); - const third = await runtime.close(); + const [first, second] = await Promise.all([runtime.close(), runtime.close()]); + const third = await runtime.close(); - expect(second).toBe(first); - expect(third).toBe(first); - await expect(runtime.closed).resolves.toBe(first); - expect(first.runners).toEqual({ stopped: ['stopped-device'], handedOff: ['handed-off-device'] }); - expect(first.sessions).toEqual({ closed: 0, failed: 0 }); - expect(first.providerReleases).toEqual({ completed: [], pending: [], unknown: [] }); - expect(first.plugins.disposed).toEqual(['stops']); - expect(first.plugins.failed).toEqual([ - { provider: 'fails', error: expect.objectContaining({ message: 'provider refused to stop' }) }, - ]); - expect(stop).toHaveBeenCalledOnce(); - expect(failing).toHaveBeenCalledOnce(); + expect(second).toBe(first); + expect(third).toBe(first); + await expect(runtime.closed).resolves.toBe(first); + expect(first.runners).toEqual({ + stopped: ['stopped-device'], + handedOff: ['handed-off-device'], + }); + expect(first.sessions).toEqual({ closed: 0, failed: 0 }); + expect(first.providerReleases).toEqual({ completed: [], pending: [], unknown: [] }); + expect(first.plugins.disposed).toEqual(['stops']); + expect(first.plugins.failed).toEqual([ + { + provider: 'fails', + error: expect.objectContaining({ message: 'provider refused to stop' }), + }, + ]); + expect(stop).toHaveBeenCalledOnce(); + expect(failing).toHaveBeenCalledOnce(); + } finally { + await runtime.close(); + } }); test('a startup that fails after acquiring resources rolls them back once', async () => { @@ -93,3 +105,33 @@ test('a startup that fails after acquiring resources rolls them back once', asyn expect(clearLock).toHaveBeenCalledOnce(); expect(process.listenerCount('SIGTERM')).toBe(signalListeners); }); + +test('a shutdown step that rejects still releases the registration, lock and providers', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-close-failure-'); + const stop = vi.fn(async () => {}); + const clearLock = vi.fn(async () => {}); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + providerDeviceRuntimes: [providerRuntime('stops', stop)], + ownerLifecycle: platformOwnerLifecycleFixture({ clearDaemonLockConfiguration: clearLock }), + deviceRuntimeGateway: Object.freeze({ + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture({ + finalizeDaemonShutdown: async () => { + throw new Error('runner finalization failed'); + }, + }), + }), + }), + }); + const paths = resolveDaemonPaths(stateDir); + + await expect(runtime.close()).rejects.toThrow('runner finalization failed'); + await expect(runtime.closed).rejects.toThrow('runner finalization failed'); + + expect(stop).toHaveBeenCalledOnce(); + expect(clearLock).toHaveBeenCalledOnce(); + expect(fs.existsSync(paths.lockPath)).toBe(false); +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts index 366f8f225e..de4829bcab 100644 --- a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts @@ -60,9 +60,10 @@ test('daemon startup installs the interactor resolution the composition supplied }); expect(runtime).not.toBeNull(); - const resolution = runtime.run(() => interactorResolution()); - expect(resolution).toBe(composed); - await resolution.resolve(IOS_SIMULATOR, {}); + await runtime.run(async () => { + expect(interactorResolution()).toBe(composed); + await interactorResolution().resolve(IOS_SIMULATOR, {}); + }); expect(composed.resolve).toHaveBeenCalledOnce(); } finally { await runtime?.close(); diff --git a/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts index b00d232616..bbf7bcfddf 100644 --- a/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts @@ -52,9 +52,11 @@ async function callSessionList(httpPort: number, token: string, requestId: strin } test('two runtimes in one process serve, close and keep their state independently', async () => { - const a = await startIsolatedRuntime('a', 'interactor-a', true); - const b = await startIsolatedRuntime('b', 'interactor-b', false); + let a: Awaited> | undefined; + let b: Awaited> | undefined; try { + a = await startIsolatedRuntime('a', 'interactor-a', true); + b = await startIsolatedRuntime('b', 'interactor-b', false); expect(a.httpPort).not.toBe(b.httpPort); expect(path.basename(a.stateDir)).not.toBe(path.basename(b.stateDir)); @@ -98,7 +100,7 @@ test('two runtimes in one process serve, close and keep their state independentl const reportB = await b.runtime.close(); expect(reportB.plugins).toEqual({ disposed: [], failed: [] }); } finally { - await a.runtime.close(); - await b.runtime.close(); + await a?.runtime.close(); + await b?.runtime.close(); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index ce4990409e..25b163635f 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -933,7 +933,9 @@ async function startDaemonRuntimeOnHost( // cannot start, PNG processing falls back to the in-process sync path. const releasePngWorker = retainPngWorker(); - const closeOnHost = async (closeOptions: { cause?: unknown }): Promise => { + let registrationFinished = false; + let lockConfigurationCleared = false; + const closeSteps = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); sessionIdleExpiry.cancel(); leaseSessionEnd.cancel(); @@ -999,14 +1001,15 @@ async function startDaemonRuntimeOnHost( releasePngWorker().catch(() => {}), sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), ]); + registrationFinished = true; await finishDaemonRegistration({ registration, infoPath, logPath, outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, }); + lockConfigurationCleared = true; await deps.ownerLifecycle.clearDaemonLockConfiguration(); - runtimeContext.dispose(); const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); return { providerReleases: { @@ -1022,6 +1025,24 @@ async function startDaemonRuntimeOnHost( }; }; + // A step that rejects must not strand what the later steps release: the PNG worker hold, the + // provider runtimes, the registration and lock, and the runtime's own state. + const closeOnHost = async (closeOptions: { cause?: unknown }): Promise => { + try { + return await closeSteps(closeOptions); + } finally { + await Promise.allSettled([ + releasePngWorker(), + pluginDisposal.dispose(), + registrationFinished + ? undefined + : finishDaemonRegistration({ registration, infoPath, logPath }), + lockConfigurationCleared ? undefined : deps.ownerLifecycle.clearDaemonLockConfiguration(), + ]); + runtimeContext.dispose(); + } + }; + // Idle reap, external callers and the executable's signal handlers all close through here, and // none of them runs inside the startup scope, so the close re-enters it. let closing: Promise | undefined; diff --git a/packages/daemon/src/server/resumable-upload.ts b/packages/daemon/src/server/resumable-upload.ts index e39d1bd28b..d7918ca88e 100644 --- a/packages/daemon/src/server/resumable-upload.ts +++ b/packages/daemon/src/server/resumable-upload.ts @@ -45,7 +45,14 @@ export function createResumableUploadRuntimeState(): RuntimeStateHandle { run: (fn) => resumableUploadStorage.run(owned, fn), enter: () => resumableUploadStorage.enterWith(owned), dispose: () => { - for (const entry of owned.byId.values()) clearTimeout(entry.timer); + for (const entry of owned.byId.values()) { + clearTimeout(entry.timer); + try { + fs.rmSync(entry.tempDir, { recursive: true, force: true }); + } catch { + // best-effort cleanup only + } + } owned.byId.clear(); owned.byKey.clear(); }, diff --git a/packages/daemon/src/sessions/materialized-path-registry.ts b/packages/daemon/src/sessions/materialized-path-registry.ts index b2d7f15137..a350d054f1 100644 --- a/packages/daemon/src/sessions/materialized-path-registry.ts +++ b/packages/daemon/src/sessions/materialized-path-registry.ts @@ -1,6 +1,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import crypto from 'node:crypto'; +import { rmSync } from 'node:fs'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -47,7 +48,14 @@ export function createMaterializedPathRuntimeState(): RuntimeStateHandle { run: (fn) => retainedPathsStorage.run(owned, fn), enter: () => retainedPathsStorage.enterWith(owned), dispose: () => { - for (const entry of owned.values()) clearTimeout(entry.timer); + for (const entry of owned.values()) { + clearTimeout(entry.timer); + try { + rmSync(entry.rootPath, { recursive: true, force: true }); + } catch { + // best-effort cleanup only + } + } owned.clear(); }, }; diff --git a/src/daemon.ts b/src/daemon.ts index 12a39b535d..d5ad7b8c64 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -16,9 +16,8 @@ async function runDaemon(): Promise { let exitCode = 0; const closeAndExit = (code: number, cause?: unknown): void => { exitCode = Math.max(exitCode, code); - void runtime.close({ cause }).catch(() => { - exitCode = 1; - }); + // A failed close is reported once, through `closed` below. + void runtime.close({ cause }).catch(() => {}); }; const failFatally = (error: unknown): void => { process.stderr.write(`Daemon error: ${asAppError(error).message}\n`); @@ -31,7 +30,8 @@ async function runDaemon(): Promise { process.on('unhandledRejection', (reason) => failFatally(reason instanceof Error ? reason : new Error(String(reason))), ); - await runtime.closed.catch(() => { + await runtime.closed.catch((error: unknown) => { + process.stderr.write(`Daemon shutdown error: ${asAppError(error).message}\n`); exitCode = 1; }); process.exit(exitCode); From c0477765dc6dff48127f821324172d24ea9e914e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 02:41:29 +0200 Subject: [PATCH 06/11] fix(daemon): assert registration release on a failed close and share one removal helper Co-Authored-By: Claude Sonnet 5.5 --- .../daemon/src/resources/artifact-tracking.ts | 16 +++------------- .../src/server/http/daemon-runtime-close.test.ts | 1 + 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/daemon/src/resources/artifact-tracking.ts b/packages/daemon/src/resources/artifact-tracking.ts index f0c827bcf1..b0062eac7d 100644 --- a/packages/daemon/src/resources/artifact-tracking.ts +++ b/packages/daemon/src/resources/artifact-tracking.ts @@ -337,12 +337,7 @@ export function cleanupDownloadableArtifact(artifactId: string): void { clearTimeout(entry.timer); pendingArtifacts().delete(artifactId); cleanupDirectoryArchive(entry); - if (!entry.deleteAfterDownload) return; - try { - fs.rmSync(entry.artifactPath, { recursive: true, force: true }); - } catch { - // best-effort cleanup only - } + if (entry.deleteAfterDownload) removeBestEffort(entry.artifactPath); } function removeBestEffort(targetPath: string): void { @@ -355,13 +350,8 @@ function removeBestEffort(targetPath: string): void { function cleanupDirectoryArchive(entry: ArtifactEntry): void { if (!entry.directoryArchive) return; - try { - fs.rmSync(entry.directoryArchive.tempDir, { recursive: true, force: true }); - } catch { - // best-effort cleanup only - } finally { - entry.directoryArchive = undefined; - } + removeBestEffort(entry.directoryArchive.tempDir); + entry.directoryArchive = undefined; } // --- Upload artifact tracking --- diff --git a/packages/daemon/src/server/http/daemon-runtime-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-close.test.ts index 0b30835d26..f9c6a8c1e1 100644 --- a/packages/daemon/src/server/http/daemon-runtime-close.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-close.test.ts @@ -134,4 +134,5 @@ test('a shutdown step that rejects still releases the registration, lock and pro expect(stop).toHaveBeenCalledOnce(); expect(clearLock).toHaveBeenCalledOnce(); expect(fs.existsSync(paths.lockPath)).toBe(false); + expect(fs.existsSync(paths.infoPath)).toBe(false); }); From 22cea00781dc036db56cfecd7af287263c84e769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 07:17:09 +0200 Subject: [PATCH 07/11] fix(apple): scope start admissions, prep builds and bad-artifact marks to the runtime Closing one daemon runtime no longer retires another runtime's start admissions or signals its prep builds. Co-Authored-By: Claude Sonnet 5.5 --- .../runner-session-runtime-isolation.test.ts | 61 +++++++++++++++++++ .../src/runner/runner-artifact.ts | 36 +++++++---- .../platform-apple/src/runner/runner-cache.ts | 11 ++-- 3 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts new file mode 100644 index 0000000000..6d0e3db2c7 --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { beforeEach, test, vi } from 'vitest'; +import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; +import { createAppleRunnerRuntimeState } from '../../core/runner-owner-state.ts'; +import { appleRunnerTestHost } from '../test-host.ts'; +import { registerRunnerPrepProcess } from '../runner-artifact.ts'; +import { stopAllIosRunnerSessions } from '../runner-session.ts'; +import { openRunnerStartAdmission, runnerPrepProcessChildren } from '../runner-xctestrun.ts'; + +const mockSignalProcessGroupBestEffort = vi.fn(); +const mockRunAppleToolCommand = vi.fn(); + +beforeEach(() => { + vi.resetAllMocks(); + mockRunAppleToolCommand.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); + appleRunnerTestHost.update({ + runAppleToolCommand: mockRunAppleToolCommand, + signalProcessGroupBestEffort: mockSignalProcessGroupBestEffort, + signalPidsBestEffort: vi.fn(), + }); +}); + +function prepChild(pid: number): ExecBackgroundResult['child'] { + return Object.assign(new EventEmitter(), { + pid, + exitCode: null, + }) as ExecBackgroundResult['child']; +} + +test("stopping every runner of one runtime leaves another runtime's prep build and admission alone", async () => { + const runtimeA = createAppleRunnerRuntimeState(); + const runtimeB = createAppleRunnerRuntimeState(); + const deviceId = 'runner-runtime-isolation-sim'; + const buildA = prepChild(6101); + const buildB = prepChild(6102); + + runtimeA.run(() => { + registerRunnerPrepProcess(deviceId, buildA, openRunnerStartAdmission(deviceId)); + }); + const admissionB = runtimeB.run(() => { + const admission = openRunnerStartAdmission(deviceId); + registerRunnerPrepProcess(deviceId, buildB, admission); + return admission; + }); + + await runtimeA.run(async () => await stopAllIosRunnerSessions()); + + const signaled = mockSignalProcessGroupBestEffort.mock.calls.map(([pid]) => pid as number); + assert.ok(signaled.includes(6101), 'runtime A stopped its own build'); + assert.ok(!signaled.includes(6102), "runtime A never signaled runtime B's build"); + assert.equal( + runtimeA.run(() => runnerPrepProcessChildren(deviceId).length), + 0, + ); + assert.equal( + runtimeB.run(() => runnerPrepProcessChildren(deviceId).length), + 1, + ); + assert.equal(admissionB.admitted, true, "runtime B's start admission stays open"); +}); diff --git a/packages/platform-apple/src/runner/runner-artifact.ts b/packages/platform-apple/src/runner/runner-artifact.ts index 2db0996401..dbf3b4a6df 100644 --- a/packages/platform-apple/src/runner/runner-artifact.ts +++ b/packages/platform-apple/src/runner/runner-artifact.ts @@ -15,6 +15,7 @@ import { emitRequestProgress, findProjectRoot, isCommandTimeoutError, + runnerRuntimeSlot, } from './host.ts'; import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -119,7 +120,10 @@ type DeviceStartState = { inFlight: Set; }; -const deviceStartStates = new Map(); +const DEVICE_START_STATES_SLOT = Symbol('apple-runner-device-start-states'); +// Per daemon runtime: a runtime's teardown retires only the start admissions it opened. +const deviceStartStates = () => + runnerRuntimeSlot(DEVICE_START_STATES_SLOT, () => new Map()); /** * Opens the admission one start answers to, taken when a caller asks for a runner. Registration is @@ -144,7 +148,7 @@ export function openRunnerStartLoopAdmission(deviceId: string): RunnerStartAdmis /** The start settled: its token leaves the device's in-flight set. Verdicts live on the token. */ export function finishRunnerStartAdmission(admission: RunnerStartAdmission): void { (admission as StartAdmission).retryPending = false; - const state = deviceStartStates.get(admission.deviceId); + const state = deviceStartStates().get(admission.deviceId); if (!state) return; state.inFlight.delete(admission as StartAdmission); forgetDeviceStartState(admission.deviceId, state); @@ -180,7 +184,10 @@ export function fenceRunnerStartAdmissionsForTeardown(deviceId: string): () => v * build past the sweep that is about to stop it. */ export function retireAllRunnerStartAdmissions(additionalDeviceIds?: Iterable): () => void { - const deviceIds = new Set([...deviceStartStates.keys(), ...(additionalDeviceIds ?? [])]); + const deviceIds = new Set([ + ...deviceStartStates().keys(), + ...(additionalDeviceIds ?? []), + ]); const settles = [...deviceIds].map((deviceId) => fenceRunnerStartAdmissionsForTeardown(deviceId)); return () => { for (const settle of settles.splice(0)) settle(); @@ -193,7 +200,7 @@ export function retireAllRunnerStartAdmissions(additionalDeviceIds?: Iterable 0; + return (deviceStartStates().get(deviceId)?.pendingTeardowns ?? 0) > 0; } /** @@ -299,17 +306,17 @@ export function releaseRunnerStartWaiter( } function stateForDevice(deviceId: string): DeviceStartState { - const existing = deviceStartStates.get(deviceId); + const existing = deviceStartStates().get(deviceId); if (existing) return existing; const state: DeviceStartState = { pendingTeardowns: 0, inFlight: new Set() }; - deviceStartStates.set(deviceId, state); + deviceStartStates().set(deviceId, state); return state; } /** Nothing fences the device and no start is in flight on it once the last of both is gone. */ function forgetDeviceStartState(deviceId: string, state: DeviceStartState): void { if (state.pendingTeardowns <= 0 && state.inFlight.size === 0) { - deviceStartStates.delete(deviceId); + deviceStartStates().delete(deviceId); } } @@ -360,7 +367,10 @@ type RunnerPrepProcess = Readonly<{ child: ExecBackgroundResult['child']; }>; -const runnerPrepProcessLedger = new Set(); +const RUNNER_PREP_LEDGER_SLOT = Symbol('apple-runner-prep-process-ledger'); +// Per daemon runtime: a runtime's stop signals only the prep children it started. +const runnerPrepProcessLedger = () => + runnerRuntimeSlot(RUNNER_PREP_LEDGER_SLOT, () => new Set()); /** * Records a prep subprocess (`xcodebuild build-for-testing`) against the device it builds for and @@ -377,9 +387,9 @@ export function registerRunnerPrepProcess( startAdmission?: RunnerStartAdmission, ): void { const entry: RunnerPrepProcess = { deviceId, startAdmission, child }; - runnerPrepProcessLedger.add(entry); + runnerPrepProcessLedger().add(entry); child.on('close', () => { - runnerPrepProcessLedger.delete(entry); + runnerPrepProcessLedger().delete(entry); }); } @@ -426,14 +436,14 @@ export function markRunnerStartRetryPending(admission: RunnerStartAdmission): vo } function prepProcessEntries(deviceId?: string): readonly RunnerPrepProcess[] { - return [...runnerPrepProcessLedger].filter( + return [...runnerPrepProcessLedger()].filter( (entry) => deviceId === undefined || entry.deviceId === deviceId, ); } export function forgetRunnerPrepProcess(child: ExecBackgroundResult['child']): void { - for (const entry of runnerPrepProcessLedger) { - if (entry.child === child) runnerPrepProcessLedger.delete(entry); + for (const entry of runnerPrepProcessLedger()) { + if (entry.child === child) runnerPrepProcessLedger().delete(entry); } } diff --git a/packages/platform-apple/src/runner/runner-cache.ts b/packages/platform-apple/src/runner/runner-cache.ts index fbb60aba68..d1d8566886 100644 --- a/packages/platform-apple/src/runner/runner-cache.ts +++ b/packages/platform-apple/src/runner/runner-cache.ts @@ -8,6 +8,7 @@ import { withProcessLock, isEnvTruthy, findProjectRoot, + runnerRuntimeSlot, } from './host.ts'; import { RUNNER_CACHE_METADATA_FILE, @@ -38,7 +39,9 @@ export { requireRunnerPhaseRemainingMs, type RunnerPhaseBudget } from './runner- const RUNNER_XCTESTRUN_CACHE_LOCK_TIMEOUT_MS = 10 * 60_000; const RUNNER_XCTESTRUN_CACHE_LOCK_POLL_MS = 100; -const badRunnerArtifactsForRun = new Set(); +const BAD_ARTIFACTS_SLOT = Symbol('apple-runner-bad-artifacts'); +const badRunnerArtifactsForRun = () => + runnerRuntimeSlot(BAD_ARTIFACTS_SLOT, () => new Set()); export type RunnerXctestrunCacheKind = 'exact' | 'miss' | 'external'; @@ -102,7 +105,7 @@ export async function markRunnerXctestrunArtifactBadForRun( return; } - badRunnerArtifactsForRun.add(artifact.derived); + badRunnerArtifactsForRun().add(artifact.derived); await withProcessLock({ acquire: () => acquireRunnerXctestrunCacheLock(artifact.derived), task: async () => { @@ -139,7 +142,7 @@ function resolveRunnerXctestrunCacheLockPath(derived: string): string { } export function cleanRunnerDerivedBeforeEvaluation(derived: string, forceRebuild: boolean): void { - if (!shouldCleanDerived() && !forceRebuild && !badRunnerArtifactsForRun.has(derived)) { + if (!shouldCleanDerived() && !forceRebuild && !badRunnerArtifactsForRun().has(derived)) { return; } emitRunnerXctestrunDecision('clean', forceRebuild ? 'forced_rebuild' : 'forced_clean', { @@ -147,7 +150,7 @@ export function cleanRunnerDerivedBeforeEvaluation(derived: string, forceRebuild }); assertSafeDerivedCleanup(derived); cleanRunnerDerivedArtifacts(derived); - badRunnerArtifactsForRun.delete(derived); + badRunnerArtifactsForRun().delete(derived); } /** From 5c98812a107ee7e9d3431603593d4605b1be80cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 07:22:36 +0200 Subject: [PATCH 08/11] refactor(move): keep daemon-runtime.ts under the size limit The runtime context, the per-runtime state list and the session teardown helpers move beside the other shutdown mechanics. Co-Authored-By: Claude Sonnet 5.5 --- .../__tests__/daemon-runtime-app-log.test.ts | 6 +- .../http/daemon-runtime-device-claims.test.ts | 3 +- .../daemon-runtime-lifecycle-shutdown.test.ts | 3 +- .../daemon-runtime-recording-teardown.test.ts | 3 +- .../daemon-runtime-snapshot-shutdown.test.ts | 3 +- .../daemon-runtime-web-close-teardown.test.ts | 3 +- .../daemon/src/server/http/daemon-runtime.ts | 166 +----------------- .../daemon/src/server/http/server-shutdown.ts | 162 +++++++++++++++++ 8 files changed, 183 insertions(+), 166 deletions(-) diff --git a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts index 5cea76642e..a102b43a0f 100644 --- a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts +++ b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts @@ -1,4 +1,5 @@ import { appLogResourceStore } from '@agent-device/capture-kit/app-log-durable-resource'; +import { teardownDaemonSessionForShutdown } from '../http/server-shutdown.ts'; import { recoverAppLogResourcesAfterDaemonLock } from '@agent-device/capture-kit/app-log-resource-recovery'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import fs from 'node:fs'; @@ -9,10 +10,7 @@ import type { ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import { createDurableResourceEnvelope } from '@agent-device/capture-kit'; import { createTestAppLogLiveHandle, mkdtempForTestSync } from '@agent-device/test-kit'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; -import { - flushDaemonStartupDiagnostics, - teardownDaemonSessionForShutdown, -} from '../http/daemon-runtime.ts'; +import { flushDaemonStartupDiagnostics } from '../http/daemon-runtime.ts'; import type { SessionState } from '../../core/session-state.ts'; import { unavailableDeviceRuntimeGateway } from '../../__tests__/support/unavailable-device-runtime-gateway.ts'; diff --git a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts index 8e860c4ed2..d886aaf892 100644 --- a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -16,7 +17,7 @@ vi.mock('../../sessions/session-teardown.ts', async (importOriginal) => { import { SessionStore } from '../../core/session-store.ts'; import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import type { SessionState } from '../../core/session-state.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; +import {} from './daemon-runtime.ts'; const mockTeardownSessionResources = vi.mocked(teardownSessionResources); const roots: string[] = []; diff --git a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts index 5449f86328..cc162432c2 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts @@ -4,6 +4,7 @@ import { mkdtempForTestSync, createTestHostEnvironment, } from '@agent-device/test-kit'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import fs from 'node:fs'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; @@ -100,7 +101,7 @@ import { daemonRuntimeDepsFixture, } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; -import { startDaemonRuntime, teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; import { makeIosSession } from '../../__tests__/support/session-factories.ts'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; diff --git a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts index f8b95f5e83..bd0bf29862 100644 --- a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts @@ -1,11 +1,12 @@ import path from 'node:path'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import { afterEach, expect, test, vi } from 'vitest'; import type { ScreenRecordingCompletion } from '@agent-device/contracts/screen-recording-runtime'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { SessionStore } from '../../core/session-store.ts'; import { makeRecordingSession } from '../../__tests__/support/session-teardown.fixtures.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; +import {} from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts b/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts index 2616d2d753..1cb9869d66 100644 --- a/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts @@ -1,4 +1,5 @@ import { afterEach, expect, test, vi } from 'vitest'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import { clearRequestAbortRegistration, markRequestCanceled, @@ -12,7 +13,7 @@ import { snapshotRuntimeFixture } from '../../__tests__/support/snapshot-runtime import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { DAEMON_SESSION_TEARDOWN_TIMEOUT_MS } from '../../sessions/session-teardown-budget.ts'; import { dispatchSnapshotViaRuntime } from '../../sessions/snapshot-runtime.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; +import {} from './daemon-runtime.ts'; import { forwardingCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; vi.mock('@agent-device/device-selection/dispatch-resolve', () => ({ diff --git a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts index 46e2b2d332..96068f105b 100644 --- a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts @@ -1,10 +1,11 @@ import path from 'node:path'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import { afterEach, expect, test, vi } from 'vitest'; import { mkdtempForTestSync, WEB_DESKTOP_DEVICE } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { makeWebSession } from '../../__tests__/support/session-teardown.fixtures.ts'; import { SessionStore } from '../../core/session-store.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; +import {} from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 25b163635f..b63cf25208 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -13,14 +13,8 @@ import { resolveDaemonServerMode, } from '@agent-device/daemon-contracts/daemon-resolution'; import { createDaemonHttpServer } from './http-server.ts'; +import { trackDownloadableArtifact } from '../../resources/artifact-tracking.ts'; import { - createArtifactTrackingRuntimeState, - trackDownloadableArtifact, -} from '../../resources/artifact-tracking.ts'; -import { createResumableUploadRuntimeState } from '../resumable-upload.ts'; -import { createMaterializedPathRuntimeState } from '../../sessions/materialized-path-registry.ts'; -import { - createProviderDeviceAdmissionRuntimeState, installProviderDeviceAdmission, type ProviderDeviceAdmission, } from '../../core/provider-device-admission.ts'; @@ -30,7 +24,6 @@ import { } from '@agent-device/daemon-contracts/daemon-policy-file'; import { assertDaemonPolicyAllowsCapability } from '../../core/daemon-policy.ts'; import { - createInteractorResolutionRuntimeState, installInteractorResolution, type InteractorResolution, } from '../../core/interactor-resolution.ts'; @@ -42,20 +35,23 @@ import { getLeaseRegistryExecutionLocks, } from '../request-execution-scope.ts'; import type { LeaseEndReason } from '../../sessions/lease-lifecycle.ts'; -import { stopSessionAppLog, teardownSessionResources } from '../../sessions/session-teardown.ts'; +import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; import { finalizeDaemonSessionApplicationLifecycle } from '../application-lifecycle-recovery.ts'; import { runtimeHintValues } from '../../sessions/session-runtime.ts'; import { closeDaemonServers, + createRuntimeContext, + teardownDaemonSessionForShutdown, + type RuntimeContext, + type WritableOutput, createProviderRuntimeDisposal, createStartupRollback, type ShutdownReport, type StartupRollback, } from './server-shutdown.ts'; import type { DaemonInvokeFn } from '../../core/daemon-request.ts'; -import type { SessionRef, SessionState } from '../../core/session-state.ts'; -import type { RuntimeHintValues } from '@agent-device/contracts/application-lifecycle-runtime'; +import type { SessionRef } from '../../core/session-state.ts'; import { createDaemonIdleReap } from '../../sessions/server/daemon-idle-reap.ts'; import { withDaemonDiagnosticsScope } from '@agent-device/daemon-contracts/daemon-diagnostics-scope'; import { createSessionIdleExpiry } from './daemon-session-idle-expiry.ts'; @@ -71,10 +67,7 @@ import { createAudioProbeAdmissionLedger } from '@agent-device/capture-kit/audio import { createPerfCaptureAdmissionLedger } from '@agent-device/capture-kit/perf-capture-admission-ledger'; import { createScreenRecordingAdmissionLedger } from '@agent-device/capture-kit/screen-recording-admission-ledger'; import { withCommandHost } from '@agent-device/host-kit/command'; -import { - createRequestCancelRuntimeState, - type RuntimeStateHandle, -} from '@agent-device/host-kit/request'; +import { type RuntimeStateHandle } from '@agent-device/host-kit/request'; import { emitDiagnostic, type ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; import { @@ -107,7 +100,6 @@ import { openWebSessionNames } from '../../sessions/web-session-names.ts'; import { createAppLogAdmissionLedger } from '../../resources/app-log-admission-ledger.ts'; import type { AppLogSessionArtifacts } from '@agent-device/contracts/app-log-runtime'; import type { PlatformOwnerLifecycle } from '../../core/platform-owner-lifecycle.ts'; -import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts'; import type { LeaseLifecycleProvider, ProviderDeviceRuntime, @@ -124,10 +116,6 @@ const DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS = 2_000; // (#2170). Bound the grace to the recorder purpose, so daemon startup stays under the client budget. const DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS = 5_000; -type WritableOutput = { - write: (chunk: string) => unknown; -}; - /** * Every capability the daemon runtime cannot construct for itself: the platform and provider * runtimes, host diagnostics, credential fingerprinting, resource cleanup, the platform lifecycle @@ -322,146 +310,10 @@ function armDaemonMetadataLossWatch( }); } -async function settleDaemonTeardownStep(params: { - session: SessionState; - stderr: WritableOutput; - resource: 'app-log' | 'session' | 'lifecycle'; - teardown: () => Promise; -}): Promise { - const { session, stderr, resource, teardown } = params; - try { - await teardown(); - return true; - } catch (error) { - stderr.write( - `Daemon ${resource} teardown error (${session.name}): ${ - error instanceof Error ? error.message : String(error) - }\n`, - ); - return false; - } -} - -/** - * Daemon-shutdown teardown of one session: bounded resource cleanup (budget - * from {@link resolveDaemonSessionTeardownTimeoutMs}, resolved BEFORE cleanup - * starts since finalizing the recording detaches `session.screenRecording`), then the - * repair-commit finalization and session deletion. Cleanup failures — including - * a recorder that could not be finalized — surface on stderr instead of being - * silently swallowed. - */ -export async function teardownDaemonSessionForShutdown(params: { - ref: SessionRef; - sessionStore: SessionStore; - stderr: WritableOutput; - platformCleanup: PlatformResourceCleanup; - finalizeApplicationLifecycle?: ( - session: SessionState, - runtimeHints: RuntimeHintValues, - ) => Promise; - afterSuccessfulTeardown?: (session: SessionState) => Promise; -}): Promise { - const { - ref, - sessionStore, - stderr, - platformCleanup, - finalizeApplicationLifecycle, - afterSuccessfulTeardown, - } = params; - const current = sessionStore.resolveCurrent(ref); - const session = current ?? ref.session; - const runtimeHints = runtimeHintValues( - current ? sessionStore.getRuntimeHints(ref.address) : undefined, - ); - const timeoutMs = resolveDaemonSessionTeardownTimeoutMs(session); - // The ownership-fenced app-log side effect must settle while this process - // still owns the daemon lock. It is intentionally outside the generic - // teardown race so lock release and runtime shutdown cannot overtake it. - - const appLogTeardownSucceeded = await settleDaemonTeardownStep({ - session, - stderr, - resource: 'app-log', - teardown: async () => await stopSessionAppLog({ ref, sessionStore }), - }); - const sessionAfterAppLog = sessionStore.resolveCurrent(ref) ?? session; - const teardown = (async () => { - const genericTeardownSucceeded = await settleDaemonTeardownStep({ - session, - stderr, - resource: 'session', - teardown: async () => - await teardownSessionResources({ - appLog: 'already-settled', - ref, - sessionStore, - platformCleanup, - }), - }); - const lifecycleTeardownSucceeded = finalizeApplicationLifecycle - ? await settleDaemonTeardownStep({ - session, - stderr, - resource: 'lifecycle', - teardown: async () => - await finalizeApplicationLifecycle(sessionAfterAppLog, runtimeHints), - }) - : true; - return genericTeardownSucceeded && lifecycleTeardownSucceeded; - })(); - const genericTeardownSucceeded = await Promise.race([ - teardown, - sleep(timeoutMs).then(() => { - stderr.write(`Daemon session teardown timed out (${session.name}).\n`); - return false; - }), - ]); - const teardownSucceeded = appLogTeardownSucceeded && genericTeardownSucceeded; - // ADR 0012 decision 6, R7 + commit semantics (C2/C5a): commit the healed - // `.ad` iff the repair transaction completed, else leave a bounded - // `REPAIR_SESSION_EXPIRED` tombstone for the reaped-before-finalize case. - sessionStore.finalizeRepairTeardown(ref); - if (teardownSucceeded) await afterSuccessfulTeardown?.(session); - sessionStore.retire(ref); - return teardownSucceeded; -} - -type RuntimeContext = Readonly<{ - run(fn: () => T): T; - /** Adds state the composition contributed and binds it for the rest of the start. */ - extend(handles: readonly RuntimeStateHandle[]): void; - dispose(): void; -}>; - -/** The per-runtime state of every module that keeps some, in the order it is entered. */ -function createDaemonRuntimeStates(): RuntimeStateHandle[] { - return [ - createRequestCancelRuntimeState(), - createProviderDeviceAdmissionRuntimeState(), - createInteractorResolutionRuntimeState(), - createArtifactTrackingRuntimeState(), - createResumableUploadRuntimeState(), - createMaterializedPathRuntimeState(), - ]; -} - export async function startDaemonRuntime( options: DaemonRuntimeOptions, ): Promise { - const states = createDaemonRuntimeStates(); - const runtimeContext: RuntimeContext = { - run: (fn) => states.reduceRight((next, state) => () => state.run(next), fn)(), - extend: (handles) => { - for (const handle of handles) { - states.push(handle); - handle.enter(); - } - }, - dispose: () => { - for (const state of states) state.dispose(); - }, - }; + const runtimeContext = createRuntimeContext(); return await runtimeContext.run( async () => await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { diff --git a/packages/daemon/src/server/http/server-shutdown.ts b/packages/daemon/src/server/http/server-shutdown.ts index fe7a81de2d..60249cdbf2 100644 --- a/packages/daemon/src/server/http/server-shutdown.ts +++ b/packages/daemon/src/server/http/server-shutdown.ts @@ -1,6 +1,23 @@ import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import type { DaemonServer } from './transport.ts'; +import { sleep } from '@agent-device/host-kit/retry'; +import { + createRequestCancelRuntimeState, + type RuntimeStateHandle, +} from '@agent-device/host-kit/request'; +import type { RuntimeHintValues } from '@agent-device/contracts/application-lifecycle-runtime'; +import { SessionStore } from '../../core/session-store.ts'; +import type { SessionRef, SessionState } from '../../core/session-state.ts'; +import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts'; +import { createProviderDeviceAdmissionRuntimeState } from '../../core/provider-device-admission.ts'; +import { createInteractorResolutionRuntimeState } from '../../core/interactor-resolution.ts'; +import { createArtifactTrackingRuntimeState } from '../../resources/artifact-tracking.ts'; +import { createResumableUploadRuntimeState } from '../resumable-upload.ts'; +import { createMaterializedPathRuntimeState } from '../../sessions/materialized-path-registry.ts'; +import { stopSessionAppLog, teardownSessionResources } from '../../sessions/session-teardown.ts'; +import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; +import { runtimeHintValues } from '../../sessions/session-runtime.ts'; const DAEMON_SHUTDOWN_TIMEOUT_MS = 5_000; @@ -99,3 +116,148 @@ export function createStartupRollback(): StartupRollback { }, }; } + +export type WritableOutput = { + write: (chunk: string) => unknown; +}; + +async function settleDaemonTeardownStep(params: { + session: SessionState; + stderr: WritableOutput; + resource: 'app-log' | 'session' | 'lifecycle'; + teardown: () => Promise; +}): Promise { + const { session, stderr, resource, teardown } = params; + try { + await teardown(); + return true; + } catch (error) { + stderr.write( + `Daemon ${resource} teardown error (${session.name}): ${ + error instanceof Error ? error.message : String(error) + }\n`, + ); + return false; + } +} + +/** + * Daemon-shutdown teardown of one session: bounded resource cleanup (budget + * from {@link resolveDaemonSessionTeardownTimeoutMs}, resolved BEFORE cleanup + * starts since finalizing the recording detaches `session.screenRecording`), then the + * repair-commit finalization and session deletion. Cleanup failures — including + * a recorder that could not be finalized — surface on stderr instead of being + * silently swallowed. + */ +export async function teardownDaemonSessionForShutdown(params: { + ref: SessionRef; + sessionStore: SessionStore; + stderr: WritableOutput; + platformCleanup: PlatformResourceCleanup; + finalizeApplicationLifecycle?: ( + session: SessionState, + runtimeHints: RuntimeHintValues, + ) => Promise; + afterSuccessfulTeardown?: (session: SessionState) => Promise; +}): Promise { + const { + ref, + sessionStore, + stderr, + platformCleanup, + finalizeApplicationLifecycle, + afterSuccessfulTeardown, + } = params; + const current = sessionStore.resolveCurrent(ref); + const session = current ?? ref.session; + const runtimeHints = runtimeHintValues( + current ? sessionStore.getRuntimeHints(ref.address) : undefined, + ); + const timeoutMs = resolveDaemonSessionTeardownTimeoutMs(session); + // The ownership-fenced app-log side effect must settle while this process + // still owns the daemon lock. It is intentionally outside the generic + // teardown race so lock release and runtime shutdown cannot overtake it. + + const appLogTeardownSucceeded = await settleDaemonTeardownStep({ + session, + stderr, + resource: 'app-log', + teardown: async () => await stopSessionAppLog({ ref, sessionStore }), + }); + const sessionAfterAppLog = sessionStore.resolveCurrent(ref) ?? session; + const teardown = (async () => { + const genericTeardownSucceeded = await settleDaemonTeardownStep({ + session, + stderr, + resource: 'session', + teardown: async () => + await teardownSessionResources({ + appLog: 'already-settled', + ref, + sessionStore, + platformCleanup, + }), + }); + const lifecycleTeardownSucceeded = finalizeApplicationLifecycle + ? await settleDaemonTeardownStep({ + session, + stderr, + resource: 'lifecycle', + teardown: async () => + await finalizeApplicationLifecycle(sessionAfterAppLog, runtimeHints), + }) + : true; + return genericTeardownSucceeded && lifecycleTeardownSucceeded; + })(); + const genericTeardownSucceeded = await Promise.race([ + teardown, + sleep(timeoutMs).then(() => { + stderr.write(`Daemon session teardown timed out (${session.name}).\n`); + return false; + }), + ]); + const teardownSucceeded = appLogTeardownSucceeded && genericTeardownSucceeded; + // ADR 0012 decision 6, R7 + commit semantics (C2/C5a): commit the healed + // `.ad` iff the repair transaction completed, else leave a bounded + // `REPAIR_SESSION_EXPIRED` tombstone for the reaped-before-finalize case. + sessionStore.finalizeRepairTeardown(ref); + if (teardownSucceeded) await afterSuccessfulTeardown?.(session); + sessionStore.retire(ref); + return teardownSucceeded; +} + +export type RuntimeContext = Readonly<{ + run(fn: () => T): T; + /** Adds state the composition contributed and binds it for the rest of the start. */ + extend(handles: readonly RuntimeStateHandle[]): void; + dispose(): void; +}>; + +/** The per-runtime state of every module that keeps some, in the order it is entered. */ +function createDaemonRuntimeStates(): RuntimeStateHandle[] { + return [ + createRequestCancelRuntimeState(), + createProviderDeviceAdmissionRuntimeState(), + createInteractorResolutionRuntimeState(), + createArtifactTrackingRuntimeState(), + createResumableUploadRuntimeState(), + createMaterializedPathRuntimeState(), + ]; +} + +/** The context one daemon runtime enters: every module's per-runtime state, composed in order. */ +export function createRuntimeContext(): RuntimeContext { + const states = createDaemonRuntimeStates(); + return { + run: (fn) => states.reduceRight((next, state) => () => state.run(next), fn)(), + extend: (handles) => { + for (const handle of handles) { + states.push(handle); + handle.enter(); + } + }, + dispose: () => { + for (const state of states) state.dispose(); + }, + }; +} From 2f29e994a0f38ee7667ab2491ce65aa77606fde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 09:50:30 +0200 Subject: [PATCH 09/11] test: drop the empty daemon-runtime imports left by the runtime-state migration Co-Authored-By: Claude Sonnet 5.5 --- .../daemon/src/server/http/daemon-runtime-device-claims.test.ts | 1 - .../src/server/http/daemon-runtime-recording-teardown.test.ts | 1 - .../src/server/http/daemon-runtime-snapshot-shutdown.test.ts | 1 - .../src/server/http/daemon-runtime-web-close-teardown.test.ts | 1 - 4 files changed, 4 deletions(-) diff --git a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts index d886aaf892..e56771a9c8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts @@ -17,7 +17,6 @@ vi.mock('../../sessions/session-teardown.ts', async (importOriginal) => { import { SessionStore } from '../../core/session-store.ts'; import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import type { SessionState } from '../../core/session-state.ts'; -import {} from './daemon-runtime.ts'; const mockTeardownSessionResources = vi.mocked(teardownSessionResources); const roots: string[] = []; diff --git a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts index bd0bf29862..1e790f0321 100644 --- a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts @@ -6,7 +6,6 @@ import { mkdtempForTestSync } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { SessionStore } from '../../core/session-store.ts'; import { makeRecordingSession } from '../../__tests__/support/session-teardown.fixtures.ts'; -import {} from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts b/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts index 1cb9869d66..1a91fe2346 100644 --- a/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-snapshot-shutdown.test.ts @@ -13,7 +13,6 @@ import { snapshotRuntimeFixture } from '../../__tests__/support/snapshot-runtime import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { DAEMON_SESSION_TEARDOWN_TIMEOUT_MS } from '../../sessions/session-teardown-budget.ts'; import { dispatchSnapshotViaRuntime } from '../../sessions/snapshot-runtime.ts'; -import {} from './daemon-runtime.ts'; import { forwardingCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; vi.mock('@agent-device/device-selection/dispatch-resolve', () => ({ diff --git a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts index 96068f105b..12376f23d9 100644 --- a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts @@ -5,7 +5,6 @@ import { mkdtempForTestSync, WEB_DESKTOP_DEVICE } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { makeWebSession } from '../../__tests__/support/session-teardown.fixtures.ts'; import { SessionStore } from '../../core/session-store.ts'; -import {} from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); From 7eedaf61b78f479e19e62cea31e980fd0f3352d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 10:14:14 +0200 Subject: [PATCH 10/11] test: run the Limrun public HTTP test on the runtime shutdown contract Co-Authored-By: Claude Sonnet 5.5 --- src/daemon-runtime-limrun-public-http.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/daemon-runtime-limrun-public-http.test.ts b/src/daemon-runtime-limrun-public-http.test.ts index 9ee0387c04..98ccbf07cc 100644 --- a/src/daemon-runtime-limrun-public-http.test.ts +++ b/src/daemon-runtime-limrun-public-http.test.ts @@ -19,8 +19,8 @@ const limrunIo = vi.hoisted(() => ({ const rootPorts = vi.hoisted(() => { const applicationLifecycle = { recoverStartupResources: vi.fn(async () => {}), - detachForDaemonShutdown: vi.fn(async () => {}), - finalizeDaemonShutdown: vi.fn(async () => {}), + detachForDaemonShutdown: vi.fn(async () => []), + finalizeDaemonShutdown: vi.fn(async () => []), }; return { providerComposition: vi.fn(async () => ({ runtimes: [] as unknown[], platformModules: [] })), @@ -93,8 +93,6 @@ test('public HTTP rejects Limrun uploaded-app listing and allocation before prov LIM_ANDROID_INSTANCE_ADB_URL: 'https://region.limrun.example/v1/android_test/adb', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -149,7 +147,7 @@ test('public HTTP rejects Limrun uploaded-app listing and allocation before prov assert.equal(rootPorts.createRequestPlatformProviders.mock.calls.length, 1); assert.equal(rootPorts.applicationLifecycle.recoverStartupResources.mock.calls.length, 1); } finally { - await runtime?.shutdown(); + await runtime?.close(); await limrun.shutdown(); fs.rmSync(paths.baseDir, { recursive: true, force: true }); if (runtime) From 1a86a193d593b788dd58909e99ace93550e2f6a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 11:02:47 +0200 Subject: [PATCH 11/11] fix(daemon): bound the runtime close and report work that outlives it Co-Authored-By: Claude Sonnet 5.5 --- .../http/daemon-runtime-bounded-close.test.ts | 90 +++++++++++++++++++ .../http/daemon-runtime-isolation.test.ts | 2 +- .../daemon/src/server/http/daemon-runtime.ts | 73 +++++++++++---- .../daemon/src/server/http/server-shutdown.ts | 38 ++++++-- .../host-kit/src/internal/request-cancel.ts | 5 ++ packages/host-kit/src/request.ts | 1 + 6 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts diff --git a/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts new file mode 100644 index 0000000000..4837d66852 --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts @@ -0,0 +1,90 @@ +import fs from 'node:fs'; +import { expect, test, vi } from 'vitest'; +import { createTestDeviceInventoryGateways } from '@agent-device/device-selection/inventory-gateways-fixtures'; +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { clearAndroidObservationFixture } from '../../__tests__/support/android-observation-fixture.ts'; +import { fakeCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; +import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { daemonPlatformServicesFixture } from '../../__tests__/support/platform-services-fixture.ts'; +import { providerCredentialsFixture } from '../../__tests__/support/provider-credentials-fixture.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +const silent = { stderr: { write: () => {} }, stdout: { write: () => {} } }; + +function hostFor(stateDir: string) { + return createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }); +} + +test('a request that ignores cancellation is abandoned at the drain budget and keeps the registration', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-bounded-close-'); + const paths = resolveDaemonPaths(stateDir); + const reached = vi.fn(); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + shutdownBudgets: { requestDrainMs: 50 }, + composeDeps: daemonRuntimeDepsFixture((input) => ({ + requestRouter: Object.freeze({ + deviceInventoryGateways: createTestDeviceInventoryGateways({ + local: async () => { + reached(); + return await new Promise(() => {}); + }, + }), + platformServices: daemonPlatformServicesFixture(), + androidObservation: clearAndroidObservationFixture, + providerCredentials: providerCredentialsFixture(input.host.env, input.baseDir), + createCommandSurface: fakeCreateCommandSurface, + }), + })), + }); + const hung = fetch(`http://127.0.0.1:${runtime.httpPort}/rpc`, { + method: 'POST', + headers: { authorization: `Bearer ${runtime.token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 'hung-1', + method: 'agent_device.command', + params: { command: 'devices', meta: { requestId: 'hung-1' } }, + }), + }).catch(() => undefined); + await vi.waitFor(() => expect(reached).toHaveBeenCalled()); + + const report = await runtime.close(); + + expect(report.requests).toEqual({ drained: 0, abandoned: 1 }); + expect(fs.existsSync(paths.infoPath)).toBe(true); + await hung; +}); + +test('a plugin that never finishes shutting down is reported pending at the budget', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-bounded-plugin-'); + const stuck = { + provider: 'stuck', + shutdown: () => new Promise(() => {}), + leaseLifecycle: {}, + }; + const done = { provider: 'done', shutdown: async () => {}, leaseLifecycle: {} }; + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + shutdownBudgets: { pluginMs: 50 }, + composeDeps: daemonRuntimeDepsFixture({ + providerDeviceRuntimes: [stuck, done] as unknown as ProviderDeviceRuntime[], + }), + }); + + const report = await runtime.close(); + + expect(report.plugins).toEqual({ disposed: ['done'], failed: [], pending: ['stuck'] }); + expect(report.requests).toEqual({ drained: 0, abandoned: 0 }); +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts index bbf7bcfddf..cb4333b091 100644 --- a/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts @@ -98,7 +98,7 @@ test('two runtimes in one process serve, close and keep their state independentl expect(b.runtime.run(() => isActiveProviderDevice({} as never))).toBe(false); const reportB = await b.runtime.close(); - expect(reportB.plugins).toEqual({ disposed: [], failed: [] }); + expect(reportB.plugins).toEqual({ disposed: [], failed: [], pending: [] }); } finally { await a?.runtime.close(); await b?.runtime.close(); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 2102753bec..3bb57be548 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -67,7 +67,7 @@ import { createAudioProbeAdmissionLedger } from '@agent-device/capture-kit/audio import { createPerfCaptureAdmissionLedger } from '@agent-device/capture-kit/perf-capture-admission-ledger'; import { createScreenRecordingAdmissionLedger } from '@agent-device/capture-kit/screen-recording-admission-ledger'; import { withCommandHost } from '@agent-device/host-kit/command'; -import { type RuntimeStateHandle } from '@agent-device/host-kit/request'; +import { cancelAllRequests, type RuntimeStateHandle } from '@agent-device/host-kit/request'; import { emitDiagnostic, type ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; import { @@ -111,6 +111,9 @@ const DAEMON_LEASE_RELEASE_TIMEOUT_MS = 1_000; const LEASE_EXPIRY_SWEEP_INTERVAL_MS = 1_000; const DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS = 1_000; const DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS = 2_000; +// A handler that ignores cancellation must not hold `close()` forever; past this it is reported +// abandoned and the registration stays held while the process lives. +const DAEMON_REQUEST_DRAIN_TIMEOUT_MS = 5_000; // An orphaned `simctl recordVideo` releases the host-wide recording lock only after it finishes // finalizing on SIGINT; force-killing it sooner leaves every later recording failing with EBUSY // (#2170). Bound the grace to the recorder purpose, so daemon startup stays under the client budget. @@ -207,6 +210,8 @@ export type DaemonRuntimeOptions = { stderr?: WritableOutput; /** The root's runtime composition, as a factory so the server owns paths, policy, and store. */ composeDeps: ComposeDaemonRuntimeDeps; + /** Shutdown time budgets, for hosts and tests that need other than the defaults. */ + shutdownBudgets?: { requestDrainMs?: number; pluginMs?: number }; }; export type DaemonRuntimeController = { @@ -376,7 +381,12 @@ async function startDaemonRuntimeOnHost( const daemonCodeOrigin = resolveDaemonCodeOrigin(); const daemonCodeSignature = resolveDaemonCodeSignature(); const providerDeviceRuntimes = deps.providerDeviceRuntimes; - const pluginDisposal = createProviderRuntimeDisposal(providerDeviceRuntimes); + const requestDrainTimeoutMs = + options.shutdownBudgets?.requestDrainMs ?? DAEMON_REQUEST_DRAIN_TIMEOUT_MS; + const pluginDisposal = createProviderRuntimeDisposal( + providerDeviceRuntimes, + options.shutdownBudgets?.pluginMs, + ); rollback.onRollback(async () => { await pluginDisposal.dispose(); }); @@ -787,6 +797,31 @@ async function startDaemonRuntimeOnHost( let registrationFinished = false; let lockConfigurationCleared = false; + // Work that ignored cancellation may still be running, so this process keeps the registration + // and lock it holds: releasing them would let a second daemon drive the same devices. + let abandonedRequests = 0; + const drainInFlightRequests = async (): Promise => { + const total = inFlightRequests.size; + if (total === 0) return { drained: 0, abandoned: 0 }; + cancelAllRequests(); + let deadlineTimer: ReturnType | undefined; + const outcome = await Promise.race([ + Promise.allSettled(inFlightRequests).then(() => 'drained' as const), + new Promise<'deadline'>((resolve) => { + deadlineTimer = setTimeout(() => resolve('deadline'), requestDrainTimeoutMs); + }), + ]); + clearTimeout(deadlineTimer); + abandonedRequests = outcome === 'drained' ? 0 : inFlightRequests.size; + if (abandonedRequests > 0) { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_shutdown_requests_abandoned', + data: { abandoned: abandonedRequests, budgetMs: requestDrainTimeoutMs }, + }); + } + return { drained: total - abandonedRequests, abandoned: abandonedRequests }; + }; const closeSteps = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); sessionIdleExpiry.cancel(); @@ -797,8 +832,11 @@ async function startDaemonRuntimeOnHost( if (closeOptions.cause) { await emitFatalDiagnostic(closeOptions.cause); } - await closeDaemonServers(servers); - await Promise.allSettled(inFlightRequests); + // Cancellation and the listener close run together: a connection held by a request that ignores + // cancellation is force-closed at the same budget the drain gives it. + const serversClosed = closeDaemonServers(servers, requestDrainTimeoutMs); + const requests = await drainInFlightRequests(); + await serversClosed; await leaseSessionEnd.drain(); // Hand healthy runners off before durable session teardown. The lifecycle gateway later // terminates only still-owned generations once all resources have finalized. @@ -853,15 +891,17 @@ async function startDaemonRuntimeOnHost( releasePngWorker().catch(() => {}), sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), ]); - registrationFinished = true; - await finishDaemonRegistration({ - registration, - infoPath, - logPath, - outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, - }); - lockConfigurationCleared = true; - await deps.ownerLifecycle.clearDaemonLockConfiguration(); + if (abandonedRequests === 0) { + registrationFinished = true; + await finishDaemonRegistration({ + registration, + infoPath, + logPath, + outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, + }); + lockConfigurationCleared = true; + await deps.ownerLifecycle.clearDaemonLockConfiguration(); + } const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); return { providerReleases: { @@ -871,6 +911,7 @@ async function startDaemonRuntimeOnHost( .map((lease) => lease.leaseId), unknown: [...unknownLeaseIds], }, + requests, runners: { stopped: [...stoppedRunners], handedOff: [...handedOffRunners] }, sessions, plugins, @@ -886,10 +927,12 @@ async function startDaemonRuntimeOnHost( await Promise.allSettled([ releasePngWorker(), pluginDisposal.dispose(), - registrationFinished + registrationFinished || abandonedRequests > 0 ? undefined : finishDaemonRegistration({ registration, infoPath, logPath }), - lockConfigurationCleared ? undefined : deps.ownerLifecycle.clearDaemonLockConfiguration(), + lockConfigurationCleared || abandonedRequests > 0 + ? undefined + : deps.ownerLifecycle.clearDaemonLockConfiguration(), ]); runtimeContext.dispose(); } diff --git a/packages/daemon/src/server/http/server-shutdown.ts b/packages/daemon/src/server/http/server-shutdown.ts index 60249cdbf2..42333bc7c3 100644 --- a/packages/daemon/src/server/http/server-shutdown.ts +++ b/packages/daemon/src/server/http/server-shutdown.ts @@ -20,6 +20,7 @@ import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-te import { runtimeHintValues } from '../../sessions/session-runtime.ts'; const DAEMON_SHUTDOWN_TIMEOUT_MS = 5_000; +const DAEMON_PLUGIN_SHUTDOWN_TIMEOUT_MS = 5_000; function forceCloseServer(server: DaemonServer): void { server.destroyConnections?.(); @@ -63,8 +64,18 @@ export type ShutdownReport = { providerReleases: { completed: string[]; pending: string[]; unknown: string[] }; /** Device runners, by device id: `handedOff` was left running for the next daemon. */ runners: { stopped: string[]; handedOff: string[] }; + /** + * Requests in flight at close. `abandoned` ones ignored cancellation past the drain budget and may + * still be running, so the registration stays held while this process lives. + */ + requests: { drained: number; abandoned: number }; sessions: { closed: number; failed: number }; - plugins: { disposed: string[]; failed: { provider: string; error: NormalizedError }[] }; + /** `pending` plugins were still shutting down when the budget ran out. */ + plugins: { + disposed: string[]; + failed: { provider: string; error: NormalizedError }[]; + pending: string[]; + }; }; export type ProviderRuntimeDisposal = Readonly<{ @@ -74,22 +85,35 @@ export type ProviderRuntimeDisposal = Readonly<{ export function createProviderRuntimeDisposal( runtimes: readonly Pick[], + timeoutMs: number = DAEMON_PLUGIN_SHUTDOWN_TIMEOUT_MS, ): ProviderRuntimeDisposal { let disposal: Promise | undefined; return { dispose: () => (disposal ??= (async () => { - const report: ShutdownReport['plugins'] = { disposed: [], failed: [] }; + const report: ShutdownReport['plugins'] = { disposed: [], failed: [], pending: [] }; + let deadlineTimer: ReturnType | undefined; + const deadline = new Promise<'deadline'>((resolve) => { + deadlineTimer = setTimeout(() => resolve('deadline'), timeoutMs); + }); await Promise.all( runtimes.map(async (runtime) => { - try { - await runtime.shutdown(); - report.disposed.push(runtime.provider); - } catch (error) { - report.failed.push({ provider: runtime.provider, error: normalizeError(error) }); + const settled = (async () => { + try { + await runtime.shutdown(); + report.disposed.push(runtime.provider); + } catch (error) { + report.failed.push({ provider: runtime.provider, error: normalizeError(error) }); + } + return 'settled' as const; + })(); + // A shutdown that outlives the budget keeps running; the report says so instead of waiting. + if ((await Promise.race([settled, deadline])) === 'deadline') { + report.pending.push(runtime.provider); } }), ); + clearTimeout(deadlineTimer); return report; })()), }; diff --git a/packages/host-kit/src/internal/request-cancel.ts b/packages/host-kit/src/internal/request-cancel.ts index 09a3122ea0..e66dc166d1 100644 --- a/packages/host-kit/src/internal/request-cancel.ts +++ b/packages/host-kit/src/internal/request-cancel.ts @@ -122,6 +122,11 @@ export function markRequestCanceled(requestId: string | undefined): void { requestAbortControllers().get(requestId)?.abort(createRequestCanceledError()); } +/** Aborts every request this runtime has in flight, as a canceled request. */ +export function cancelAllRequests(): void { + for (const requestId of requestAbortControllers().keys()) markRequestCanceled(requestId); +} + export function clearRequestCanceled( requestId: string | undefined, registration?: RequestAbortRegistration, diff --git a/packages/host-kit/src/request.ts b/packages/host-kit/src/request.ts index 5ae44f36fc..26463db190 100644 --- a/packages/host-kit/src/request.ts +++ b/packages/host-kit/src/request.ts @@ -1,4 +1,5 @@ export { + cancelAllRequests, clearRequestAbortRegistration, createRequestCancelRuntimeState, type RuntimeStateHandle,