diff --git a/packages/capture-kit/src/png-worker-client.test.ts b/packages/capture-kit/src/png-worker-client.test.ts index 8c9835b713..847fd5126f 100644 --- a/packages/capture-kit/src/png-worker-client.test.ts +++ b/packages/capture-kit/src/png-worker-client.test.ts @@ -9,6 +9,7 @@ import { decodePngAsync, decodeScreenshotImageAsync, encodePngAsync, + retainPngWorker, terminatePngWorker, transcodeScreenshotToPngAsync, } from './png-worker-client.ts'; @@ -225,3 +226,19 @@ test('decodeScreenshotImageAsync rejects a container it cannot read with the can }, ); }); + +test('the worker serves every holder and stops only when the last one lets go', async () => { + const first = retainPngWorker(); + const second = retainPngWorker(); + const encoded = PNG.sync.write(buildPatternPng(5, 4, 2)); + + await first(); + await first(); + const stillServing = await runOnWorkerThread('decode after one release', () => + decodePngAsync(encoded, 'fixture'), + ); + assert.equal(stillServing.width, 5); + + await second(); + assert.equal(process.getActiveResourcesInfo().includes('MessagePort'), false); +}); diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index 994e1cedf0..e66ad5aba4 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -181,9 +181,23 @@ async function runPngJob( } } -/** Daemon startup hook: spawns the worker before the first screenshot job. */ -export function prewarmPngWorker(): void { +let pngWorkerHolds = 0; + +/** + * Daemon startup hook: spawns the worker before the first screenshot job and holds it for the + * caller. The worker is shared by every runtime in the process, so it stops when the last holder + * lets go; the returned release is idempotent. + */ +export function retainPngWorker(): () => Promise { obtainWorker(); + pngWorkerHolds += 1; + let released = false; + return async () => { + if (released) return; + released = true; + pngWorkerHolds -= 1; + if (pngWorkerHolds === 0) await terminatePngWorker(); + }; } /** Stops the worker thread (used by tests and shutdown); later calls respawn it. */ diff --git a/packages/contracts/src/application-lifecycle-runtime.ts b/packages/contracts/src/application-lifecycle-runtime.ts index 8db2fb2807..cf62df9d4d 100644 --- a/packages/contracts/src/application-lifecycle-runtime.ts +++ b/packages/contracts/src/application-lifecycle-runtime.ts @@ -382,8 +382,10 @@ export type AppleApplicationTools = Readonly<{ applyRuntimeHints(device: DeviceInfo, input: RuntimeHintsApplicationInput): Promise; clearRuntimeHints(device: DeviceInfo, input: RuntimeHintsApplicationInput): Promise; dismissCloseAlerts(device: DeviceInfo, input: CloseApplicationFinalizationInput): Promise; - detachRunnerSessionsForShutdown(): Promise; - finalizeRunnerSessionsForShutdown(): Promise; + /** The devices whose runner was handed to the next daemon instead of stopped. */ + detachRunnerSessionsForShutdown(): Promise; + /** The devices whose runner this daemon stopped. */ + finalizeRunnerSessionsForShutdown(): Promise; }>; /** Lazy Android mechanics used by the Android package's own lifecycle binding. */ @@ -413,6 +415,6 @@ export type AndroidApplicationTools = Readonly<{ /** Gateway-coordinated durable phases whose concrete mechanics stay package-owned. */ export type ApplicationLifecycleResourceLifecycle = Readonly<{ recoverStartupResources(input: Readonly<{ stateDir: string }>): Promise; - detachForDaemonShutdown(): Promise; - finalizeDaemonShutdown(): Promise; + detachForDaemonShutdown(): Promise; + finalizeDaemonShutdown(): Promise; }>; diff --git a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts index c4eaed88ed..0e7cbc42ff 100644 --- a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts +++ b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts @@ -25,8 +25,8 @@ export function applicationLifecycleResourceFixture( ): ApplicationLifecycleResourceLifecycle { return Object.freeze({ recoverStartupResources: vi.fn(async () => {}), - detachForDaemonShutdown: vi.fn(async () => {}), - finalizeDaemonShutdown: vi.fn(async () => {}), + detachForDaemonShutdown: vi.fn(async (): Promise => []), + finalizeDaemonShutdown: vi.fn(async (): Promise => []), ...overrides, }); } @@ -59,6 +59,7 @@ export function daemonRuntimeDepsFixture( return async (input) => { const resolved = typeof overrides === 'function' ? overrides(input) : overrides; return Object.freeze({ + runtimeStates: [], ownedProcessRecords: createOwnedProcessRecordStore({ stateDir: input.baseDir, sessionsDir: input.sessionsDir, diff --git a/packages/daemon/src/core/interactor-resolution.ts b/packages/daemon/src/core/interactor-resolution.ts index 2584685635..61d68ac2c6 100644 --- a/packages/daemon/src/core/interactor-resolution.ts +++ b/packages/daemon/src/core/interactor-resolution.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import type { Interactor, RunnerContext } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -27,7 +29,24 @@ const NO_INTERACTOR_RESOLUTION: InteractorResolution = { }, }; -let installedInteractorResolution: InteractorResolution = NO_INTERACTOR_RESOLUTION; +type InteractorResolutionState = { resolution: InteractorResolution }; + +const interactorResolutionStorage = new AsyncLocalStorage(); +const standaloneInteractorResolution: InteractorResolutionState = { + resolution: NO_INTERACTOR_RESOLUTION, +}; +const interactorResolutionState = (): InteractorResolutionState => + interactorResolutionStorage.getStore() ?? standaloneInteractorResolution; + +/** The resolution one daemon runtime owns; the runtime installs its composition's into it. */ +export function createInteractorResolutionRuntimeState(): RuntimeStateHandle { + const owned: InteractorResolutionState = { resolution: NO_INTERACTOR_RESOLUTION }; + return { + run: (fn) => interactorResolutionStorage.run(owned, fn), + enter: () => interactorResolutionStorage.enterWith(owned), + dispose: () => {}, + }; +} /** * Root composition's installation point, beside the provider-device admission that shares the @@ -35,9 +54,9 @@ let installedInteractorResolution: InteractorResolution = NO_INTERACTOR_RESOLUTI * the module root composition imports, and the daemon never loads it itself. */ export function installInteractorResolution(resolution: InteractorResolution): void { - installedInteractorResolution = resolution; + interactorResolutionState().resolution = resolution; } export function interactorResolution(): InteractorResolution { - return installedInteractorResolution; + return interactorResolutionState().resolution; } diff --git a/packages/daemon/src/core/provider-device-admission.ts b/packages/daemon/src/core/provider-device-admission.ts index bae7032244..eadf0de88d 100644 --- a/packages/daemon/src/core/provider-device-admission.ts +++ b/packages/daemon/src/core/provider-device-admission.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import type { DeviceInfo } from '@agent-device/kernel/device'; /** @@ -18,7 +20,24 @@ const NO_PROVIDER_DEVICE_ADMISSION: ProviderDeviceAdmission = { isActive: () => false, }; -let installedProviderDeviceAdmission: ProviderDeviceAdmission = NO_PROVIDER_DEVICE_ADMISSION; +type ProviderAdmissionState = { admission: ProviderDeviceAdmission }; + +const providerAdmissionStorage = new AsyncLocalStorage(); +const standaloneProviderAdmission: ProviderAdmissionState = { + admission: NO_PROVIDER_DEVICE_ADMISSION, +}; +const providerAdmissionState = (): ProviderAdmissionState => + providerAdmissionStorage.getStore() ?? standaloneProviderAdmission; + +/** The admission one daemon runtime owns; the runtime installs its composition's into it. */ +export function createProviderDeviceAdmissionRuntimeState(): RuntimeStateHandle { + const owned: ProviderAdmissionState = { admission: NO_PROVIDER_DEVICE_ADMISSION }; + return { + run: (fn) => providerAdmissionStorage.run(owned, fn), + enter: () => providerAdmissionStorage.enterWith(owned), + dispose: () => {}, + }; +} /** * Root composition's installation point. The provider runtime scope is ambient per request, so @@ -26,9 +45,9 @@ let installedProviderDeviceAdmission: ProviderDeviceAdmission = NO_PROVIDER_DEVI * provider runtime internals itself. */ export function installProviderDeviceAdmission(admission: ProviderDeviceAdmission): void { - installedProviderDeviceAdmission = admission; + providerAdmissionState().admission = admission; } export function isActiveProviderDevice(device: DeviceInfo): boolean { - return installedProviderDeviceAdmission.isActive(device); + return providerAdmissionState().admission.isActive(device); } diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index b7ef46715a..a72685e11c 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -1,9 +1,8 @@ -export { startDaemonRuntime } from './server/http/daemon-runtime.ts'; +export { daemonStartupExitCode, startDaemonRuntime } from './server/http/daemon-runtime.ts'; export type { ComposeDaemonRuntimeDeps, DaemonRuntimeController, DaemonRuntimeDeps, - DaemonRuntimeDepsInput, } from './server/http/daemon-runtime.ts'; export type { DaemonInvokeFn, DaemonRequest } from './core/daemon-request.ts'; export type { DaemonPlatformServices } from './core/platform-services.ts'; diff --git a/packages/daemon/src/resources/artifact-tracking.test.ts b/packages/daemon/src/resources/artifact-tracking.test.ts new file mode 100644 index 0000000000..c2e4538847 --- /dev/null +++ b/packages/daemon/src/resources/artifact-tracking.test.ts @@ -0,0 +1,32 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { + createArtifactTrackingRuntimeState, + trackDownloadableArtifact, + trackUploadedArtifact, +} from './artifact-tracking.ts'; + +test('disposing a runtime removes the temporary files it still tracks', () => { + const root = mkdtempForTestSync('agent-device-artifact-dispose-'); + const artifactPath = path.join(root, 'artifact.txt'); + const uploadDir = path.join(root, 'upload'); + fs.writeFileSync(artifactPath, 'artifact'); + fs.mkdirSync(uploadDir); + fs.writeFileSync(path.join(uploadDir, 'payload'), 'payload'); + const runtime = createArtifactTrackingRuntimeState(); + + runtime.run(() => { + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'artifact.txt', + }); + trackUploadedArtifact({ artifactPath: path.join(uploadDir, 'payload'), tempDir: uploadDir }); + }); + runtime.dispose(); + + expect(fs.existsSync(uploadDir)).toBe(false); + expect(fs.existsSync(artifactPath)).toBe(false); +}); diff --git a/packages/daemon/src/resources/artifact-tracking.ts b/packages/daemon/src/resources/artifact-tracking.ts index efccad7590..b0062eac7d 100644 --- a/packages/daemon/src/resources/artifact-tracking.ts +++ b/packages/daemon/src/resources/artifact-tracking.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; @@ -44,7 +46,43 @@ type ArtifactEntry = { timer: ReturnType; }; -const pendingArtifacts = new Map(); +type ArtifactTrackingState = { + artifacts: Map; + uploads: Map; +}; + +const artifactTrackingStorage = new AsyncLocalStorage(); +const createArtifactTrackingState = (): ArtifactTrackingState => ({ + artifacts: new Map(), + uploads: new Map(), +}); +const standaloneArtifactTracking = createArtifactTrackingState(); +const artifactTracking = (): ArtifactTrackingState => + artifactTrackingStorage.getStore() ?? standaloneArtifactTracking; +const pendingArtifacts = () => artifactTracking().artifacts; +const pendingUploads = () => artifactTracking().uploads; + +/** The artifacts and uploads one daemon runtime tracks; disposing it stops their expiry timers. */ +export function createArtifactTrackingRuntimeState(): RuntimeStateHandle { + const owned = createArtifactTrackingState(); + return { + run: (fn) => artifactTrackingStorage.run(owned, fn), + enter: () => artifactTrackingStorage.enterWith(owned), + dispose: () => { + for (const entry of owned.artifacts.values()) { + clearTimeout(entry.timer); + cleanupDirectoryArchive(entry); + if (entry.deleteAfterDownload) removeBestEffort(entry.artifactPath); + } + for (const entry of owned.uploads.values()) { + clearTimeout(entry.timer); + removeBestEffort(entry.tempDir); + } + owned.artifacts.clear(); + owned.uploads.clear(); + }, + }; +} /** What a finished request registers for each artifact it hands back to the client. */ export type DownloadableArtifactRegistration = { @@ -73,7 +111,7 @@ export function trackDownloadableArtifact( cleanupDownloadableArtifact(artifactId); }, ARTIFACT_CLEANUP_TIMEOUT_MS); timer.unref(); - pendingArtifacts.set(artifactId, { + pendingArtifacts().set(artifactId, { artifactPath: params.artifactPath, tenantId: params.tenantId, artifactType: params.artifactType, @@ -105,7 +143,7 @@ export async function listDownloadableArtifacts( tenantId?: string, ): Promise { const artifacts: DaemonArtifactInventoryEntry[] = []; - for (const [id, entry] of pendingArtifacts) { + for (const [id, entry] of pendingArtifacts()) { if (!canReadArtifact(entry, tenantId)) continue; let stat: fs.Stats; try { @@ -148,7 +186,7 @@ function readDownloadableArtifactEntry( tenantId: string | undefined, ): { entry: ArtifactEntry; stat: fs.Stats } { const entry = requireTenantOwnedEntry( - pendingArtifacts, + pendingArtifacts(), artifactId, tenantId, DOWNLOADABLE_ARTIFACT_RESOURCE, @@ -249,7 +287,7 @@ async function createDirectoryArchive( sizeBytes: fs.statSync(archivePath).size, }; // The artifact may be consumed or expire while tar is still archiving. - if (pendingArtifacts.get(artifactId) !== entry) { + if (pendingArtifacts().get(artifactId) !== entry) { fs.rmSync(tempDir, { recursive: true, force: true }); throw expiredTenantOwnedEntryError(DOWNLOADABLE_ARTIFACT_RESOURCE, artifactId); } @@ -294,14 +332,17 @@ function hasUnsafeFilenameCharacter(fileName: string): boolean { } export function cleanupDownloadableArtifact(artifactId: string): void { - const entry = pendingArtifacts.get(artifactId); + const entry = pendingArtifacts().get(artifactId); if (!entry) return; clearTimeout(entry.timer); - pendingArtifacts.delete(artifactId); + pendingArtifacts().delete(artifactId); cleanupDirectoryArchive(entry); - if (!entry.deleteAfterDownload) return; + if (entry.deleteAfterDownload) removeBestEffort(entry.artifactPath); +} + +function removeBestEffort(targetPath: string): void { try { - fs.rmSync(entry.artifactPath, { recursive: true, force: true }); + fs.rmSync(targetPath, { recursive: true, force: true }); } catch { // best-effort cleanup only } @@ -309,13 +350,8 @@ export function cleanupDownloadableArtifact(artifactId: string): void { function cleanupDirectoryArchive(entry: ArtifactEntry): void { if (!entry.directoryArchive) return; - try { - fs.rmSync(entry.directoryArchive.tempDir, { recursive: true, force: true }); - } catch { - // best-effort cleanup only - } finally { - entry.directoryArchive = undefined; - } + removeBestEffort(entry.directoryArchive.tempDir); + entry.directoryArchive = undefined; } // --- Upload artifact tracking --- @@ -334,8 +370,6 @@ type UploadEntry = { timer: ReturnType; }; -const pendingUploads = new Map(); - export function trackUploadedArtifact(params: { artifactPath: string; tempDir: string; @@ -346,7 +380,7 @@ export function trackUploadedArtifact(params: { cleanupUploadedArtifact(uploadId); }, UPLOAD_CLEANUP_TIMEOUT_MS); timer.unref(); - pendingUploads.set(uploadId, { + pendingUploads().set(uploadId, { artifactPath: params.artifactPath, tempDir: params.tempDir, tenantId: params.tenantId, @@ -357,7 +391,7 @@ export function trackUploadedArtifact(params: { export function prepareUploadedArtifact(uploadId: string, tenantId?: string): string { const entry = requireTenantOwnedEntry( - pendingUploads, + pendingUploads(), uploadId, tenantId, UPLOADED_ARTIFACT_RESOURCE, @@ -367,9 +401,9 @@ export function prepareUploadedArtifact(uploadId: string, tenantId?: string): st } export function cleanupUploadedArtifact(uploadId: string): void { - const entry = pendingUploads.get(uploadId); + const entry = pendingUploads().get(uploadId); if (!entry) return; clearTimeout(entry.timer); - pendingUploads.delete(uploadId); + pendingUploads().delete(uploadId); fs.rmSync(entry.tempDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts index 532ae85cde..a102b43a0f 100644 --- a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts +++ b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts @@ -1,4 +1,5 @@ import { appLogResourceStore } from '@agent-device/capture-kit/app-log-durable-resource'; +import { teardownDaemonSessionForShutdown } from '../http/server-shutdown.ts'; import { recoverAppLogResourcesAfterDaemonLock } from '@agent-device/capture-kit/app-log-resource-recovery'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import fs from 'node:fs'; @@ -9,10 +10,7 @@ import type { ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import { createDurableResourceEnvelope } from '@agent-device/capture-kit'; import { createTestAppLogLiveHandle, mkdtempForTestSync } from '@agent-device/test-kit'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; -import { - flushDaemonStartupDiagnostics, - teardownDaemonSessionForShutdown, -} from '../http/daemon-runtime.ts'; +import { flushDaemonStartupDiagnostics } from '../http/daemon-runtime.ts'; import type { SessionState } from '../../core/session-state.ts'; import { unavailableDeviceRuntimeGateway } from '../../__tests__/support/unavailable-device-runtime-gateway.ts'; @@ -36,13 +34,12 @@ test('daemon startup configures the Apple runner owner after acquiring the lock, const source = fs.readFileSync(new URL('../http/daemon-runtime.ts', import.meta.url), 'utf8'); const acquiredLock = source.indexOf("if (acquisition.status !== 'acquired')"); const runnerOwnerConfigured = source.indexOf('await deps.ownerLifecycle.configureForDaemonLock('); - const lockFailureExit = source.indexOf('exit(', acquiredLock); + const lockFailureExit = source.indexOf("reason: 'daemon_registration_refused'", acquiredLock); expect(acquiredLock).toBeGreaterThanOrEqual(0); expect(lockFailureExit).toBeGreaterThan(acquiredLock); expect(runnerOwnerConfigured).toBeGreaterThan(acquiredLock); - // The configure call sits inside the post-lock try block, after the failure branch that exits - // for a lock held by another process. + // The configure call comes after the failure branch that refuses a lock held by another process. expect(runnerOwnerConfigured).toBeGreaterThan(lockFailureExit); }); diff --git a/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts b/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts index 5be1e1985a..0d56c8ec56 100644 --- a/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts +++ b/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts @@ -42,6 +42,7 @@ test('journals and bounds a hung recoverable lease release before the final drai await vi.advanceTimersByTimeAsync(10); await expect(drain).resolves.toEqual({ pending: [lease], + unknown: [lease], released: [], }); } finally { @@ -83,7 +84,7 @@ test('final drain joins a release that completes after the lease timeout', async const drain = expiredProviderLeaseReleaser.drain(2_000); await vi.advanceTimersByTimeAsync(500); - await expect(drain).resolves.toEqual({ pending: [], released: [lease] }); + await expect(drain).resolves.toEqual({ pending: [], unknown: [], released: [lease] }); } finally { expiredProviderLeaseReleaser.shutdown(); vi.useRealTimers(); @@ -129,7 +130,11 @@ test('a hung provider release does not starve another lease during shutdown', as expect(leaseRegistry.listActiveLeases()).toEqual([]); const drain = expiredProviderLeaseReleaser.drain(2_000); await vi.advanceTimersByTimeAsync(2_000); - await expect(drain).resolves.toEqual({ pending: [hungLease], released: [releasedLease] }); + await expect(drain).resolves.toEqual({ + pending: [hungLease], + unknown: [hungLease], + released: [releasedLease], + }); } finally { expiredProviderLeaseReleaser.shutdown(); vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts index 8e31d53d8d..16560207b9 100644 --- a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts @@ -13,11 +13,6 @@ async function downloadTrackedArtifact(env: Record, awaitConsump const stateDir = mkdtempForTestSync('agent-device-daemon-runtime-artifacts-'); const artifactPath = path.join(stateDir, 'runtime-artifact.txt'); fs.writeFileSync(artifactPath, 'runtime-artifact'); - const artifactId = trackDownloadableArtifact({ - artifactPath, - artifactType: 'runtime-artifact', - fileName: 'runtime-artifact.txt', - }); const runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { @@ -28,14 +23,18 @@ async function downloadTrackedArtifact(env: Record, awaitConsump ...env, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture(), }); + const artifactId = runtime.run(() => + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'runtime-artifact.txt', + }), + ); try { - expect(runtime).not.toBeNull(); const response = await fetch( `http://127.0.0.1:${runtime?.httpPort}/artifacts/${encodeURIComponent(artifactId)}`, { headers: { authorization: `Bearer ${runtime?.token}`, connection: 'close' } }, @@ -48,8 +47,8 @@ async function downloadTrackedArtifact(env: Record, awaitConsump } return { status: response.status, body, onDisk: fs.existsSync(artifactPath) }; } finally { - await runtime?.shutdown(); - cleanupDownloadableArtifact(artifactId); + runtime.run(() => cleanupDownloadableArtifact(artifactId)); + await runtime.close(); } } diff --git a/packages/daemon/src/server/http/daemon-runtime-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-close.test.ts new file mode 100644 index 0000000000..f9c6a8c1e1 --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-close.test.ts @@ -0,0 +1,138 @@ +import fs from 'node:fs'; +import { expect, test, vi } from 'vitest'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { + applicationLifecycleResourceFixture, + daemonRuntimeDepsFixture, + platformOwnerLifecycleFixture, +} from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +function hostFor(stateDir: string) { + return createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }); +} + +function providerRuntime(provider: string, shutdown: () => Promise): ProviderDeviceRuntime { + return { provider, shutdown, leaseLifecycle: {} } as unknown as ProviderDeviceRuntime; +} + +const silent = { stderr: { write: () => {} }, stdout: { write: () => {} } }; + +test('close is idempotent and every caller receives the one report', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-close-'); + const stop = vi.fn(async () => {}); + const failing = vi.fn(async () => { + throw new Error('provider refused to stop'); + }); + const signalListeners = process.listenerCount('SIGTERM'); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + providerDeviceRuntimes: [providerRuntime('stops', stop), providerRuntime('fails', failing)], + deviceRuntimeGateway: Object.freeze({ + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture({ + detachForDaemonShutdown: async () => ['handed-off-device'], + finalizeDaemonShutdown: async () => ['stopped-device'], + }), + }), + }), + }); + try { + expect(process.listenerCount('SIGTERM')).toBe(signalListeners); + + const [first, second] = await Promise.all([runtime.close(), runtime.close()]); + const third = await runtime.close(); + + expect(second).toBe(first); + expect(third).toBe(first); + await expect(runtime.closed).resolves.toBe(first); + expect(first.runners).toEqual({ + stopped: ['stopped-device'], + handedOff: ['handed-off-device'], + }); + expect(first.sessions).toEqual({ closed: 0, failed: 0 }); + expect(first.providerReleases).toEqual({ completed: [], pending: [], unknown: [] }); + expect(first.plugins.disposed).toEqual(['stops']); + expect(first.plugins.failed).toEqual([ + { + provider: 'fails', + error: expect.objectContaining({ message: 'provider refused to stop' }), + }, + ]); + expect(stop).toHaveBeenCalledOnce(); + expect(failing).toHaveBeenCalledOnce(); + } finally { + await runtime.close(); + } +}); + +test('a startup that fails after acquiring resources rolls them back once', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-rollback-'); + const stop = vi.fn(async () => {}); + const clearLock = vi.fn(async () => {}); + const failure = new Error('startup step failed'); + const signalListeners = process.listenerCount('SIGTERM'); + + await expect( + startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + providerDeviceRuntimes: [providerRuntime('stops', stop)], + ownerLifecycle: platformOwnerLifecycleFixture({ + configureForDaemonLock: async () => { + throw failure; + }, + clearDaemonLockConfiguration: clearLock, + }), + }), + }), + ).rejects.toBe(failure); + + expect(stop).toHaveBeenCalledOnce(); + expect(clearLock).toHaveBeenCalledOnce(); + expect(process.listenerCount('SIGTERM')).toBe(signalListeners); +}); + +test('a shutdown step that rejects still releases the registration, lock and providers', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-close-failure-'); + const stop = vi.fn(async () => {}); + const clearLock = vi.fn(async () => {}); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + providerDeviceRuntimes: [providerRuntime('stops', stop)], + ownerLifecycle: platformOwnerLifecycleFixture({ clearDaemonLockConfiguration: clearLock }), + deviceRuntimeGateway: Object.freeze({ + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture({ + finalizeDaemonShutdown: async () => { + throw new Error('runner finalization failed'); + }, + }), + }), + }), + }); + const paths = resolveDaemonPaths(stateDir); + + await expect(runtime.close()).rejects.toThrow('runner finalization failed'); + await expect(runtime.closed).rejects.toThrow('runner finalization failed'); + + expect(stop).toHaveBeenCalledOnce(); + expect(clearLock).toHaveBeenCalledOnce(); + expect(fs.existsSync(paths.lockPath)).toBe(false); + expect(fs.existsSync(paths.infoPath)).toBe(false); +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts index 98bf37f55e..de4829bcab 100644 --- a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts @@ -14,12 +14,13 @@ import { WEB_DESKTOP_DEVICE, createTestHostEnvironment, } from '@agent-device/test-kit'; -import { - installInteractorResolution, - interactorResolution, -} from '../../core/interactor-resolution.ts'; +import { interactorResolution } from '../../core/interactor-resolution.ts'; import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; -import { startDaemonRuntime, type DaemonRuntimeDepsInput } from './daemon-runtime.ts'; +import { + daemonStartupExitCode, + startDaemonRuntime, + type DaemonRuntimeDepsInput, +} from './daemon-runtime.ts'; // The server builds its own store, so capture that instance: the proof is about the store the // server hands to the composition callbacks, not one the test fabricates. @@ -42,8 +43,7 @@ vi.mock('../../core/session-store.ts', async (importOriginal) => { test('daemon startup installs the interactor resolution the composition supplied', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-interactor-composition-'); const composed = Object.freeze({ resolve: vi.fn(async () => ({}) as never) }); - const previousResolution = interactorResolution(); - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ @@ -54,20 +54,19 @@ test('daemon startup installs the interactor resolution the composition supplied AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ interactorResolution: composed }), }); expect(runtime).not.toBeNull(); - expect(interactorResolution()).toBe(composed); - await interactorResolution().resolve(IOS_SIMULATOR, {}); + await runtime.run(async () => { + expect(interactorResolution()).toBe(composed); + await interactorResolution().resolve(IOS_SIMULATOR, {}); + }); expect(composed.resolve).toHaveBeenCalledOnce(); } finally { - await runtime?.shutdown(); - installInteractorResolution(previousResolution); + await runtime?.close(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); @@ -80,11 +79,8 @@ test('a daemon attempt losing the lock shuts down every constructed provider', a throw new Error('cleanup failed'); }); const otherShutdown = vi.fn(async () => {}); - const exit = vi.fn(); - const runtime = await startDaemonRuntime({ + const refusal = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { AGENT_DEVICE_STATE_DIR: stateDir } }), - exit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ @@ -97,11 +93,10 @@ test('a daemon attempt losing the lock shuts down every constructed provider', a }) as unknown as ProviderDeviceRuntime, ), }), - }); - expect(runtime).toBeNull(); + }).catch((error: unknown) => error); + expect(daemonStartupExitCode(refusal)).toBe(DAEMON_STARTUP_EXIT_CODES.busy); expect(shutdown).toHaveBeenCalledOnce(); expect(otherShutdown).toHaveBeenCalledOnce(); - expect(exit).toHaveBeenCalledWith(DAEMON_STARTUP_EXIT_CODES.busy); await held.owner.finish(); }); @@ -119,8 +114,6 @@ test('the server runs on the host it was given and hands that host to the compos const fixtureDeps = daemonRuntimeDepsFixture(); const runtime = await startDaemonRuntime({ host, - registerProcessHandlers: false, - exit: () => {}, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: async (input) => { @@ -132,7 +125,7 @@ test('the server runs on the host it was given and hands that host to the compos assert.equal(composed?.host, host); assert.equal(composed?.baseDir, resolveDaemonPaths(stateDir).baseDir); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -150,10 +143,9 @@ test('the server derives the policy gate and session callbacks and hands them to JSON.stringify({ version: 1, capabilities: { deny: ['device-shutdown'] } }), ); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; let composed: DaemonRuntimeDepsInput | undefined; const fixtureDeps = daemonRuntimeDepsFixture(); - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ @@ -164,8 +156,6 @@ test('the server derives the policy gate and session callbacks and hands them to AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: async (input) => { @@ -173,8 +163,6 @@ test('the server derives the policy gate and session callbacks and hands them to return await fixtureDeps(input); }, }); - assert.notEqual(runtime, null, 'the assembly must boot'); - assert.deepEqual(exits, []); assert.ok(composed, 'the server must call the factory'); // The server resolved the paths and loaded the operator's policy before composing anything. assert.equal(composed.baseDir, paths.baseDir); @@ -206,11 +194,10 @@ test('the server derives the policy gate and session callbacks and hands them to outputPath: store.resolveAppLogPath('web-assembly'), pidPath: store.resolveAppLogPidPath('web-assembly'), }); - await runtime?.shutdown(); + await runtime?.close(); runtime = null; - assert.deepEqual(exits, [0]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -220,12 +207,9 @@ test('the server derives the policy gate and session callbacks and hands them to */ test('an unreadable policy stops the assembly before the factory is called', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-assembly-bad-policy-'); - const exits: number[] = []; - const stderr: string[] = []; let factoryCalls = 0; - let runtime: Awaited> = null; try { - runtime = await startDaemonRuntime({ + const refusal = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { ...process.env, @@ -234,20 +218,17 @@ test('an unreadable policy stops the assembly before the factory is called', asy AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - stderr: { write: (chunk) => stderr.push(chunk) }, + stderr: { write: () => {} }, stdout: { write: () => {} }, - registerProcessHandlers: false, composeDeps: async (input) => { factoryCalls += 1; return await daemonRuntimeDepsFixture()(input); }, - }); - assert.equal(runtime, null); - assert.deepEqual(exits, [1]); + }).catch((error: unknown) => error); + assert.equal(daemonStartupExitCode(refusal), 1); assert.equal(factoryCalls, 0, 'no runtime may be built under an unreadable policy'); - assert.match(stderr.join(''), /Daemon error: .*/); + assert.match((refusal as Error).message, /policy/i); } finally { - await runtime?.shutdown(); + fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts index 8e860c4ed2..e56771a9c8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -16,7 +17,6 @@ vi.mock('../../sessions/session-teardown.ts', async (importOriginal) => { import { SessionStore } from '../../core/session-store.ts'; import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import type { SessionState } from '../../core/session-state.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; const mockTeardownSessionResources = vi.mocked(teardownSessionResources); const roots: string[] = []; diff --git a/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts b/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts index af6061195a..f27d9d84f7 100644 --- a/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts @@ -30,11 +30,7 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-rt-'); const paths = resolveDaemonPaths(stateDir); - let exitCode: number | undefined; - let resolveExit: () => void; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -47,25 +43,22 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '80', }, }), - exit: (code) => { - exitCode = code; - resolveExit(); - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); assert.ok(fs.existsSync(paths.lockPath), 'daemon lock should be held right after startup'); - assert.equal(exitCode, undefined); + assert.equal(closed, false); await vi.advanceTimersByTimeAsync(79); - assert.equal(exitCode, undefined); + assert.equal(closed, false); await vi.advanceTimersByTimeAsync(1); await vi.runOnlyPendingTimersAsync(); await exited; - assert.equal(exitCode, 0); + assert.equal(closed, true); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -76,7 +69,7 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS is 0', async () => { vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-off-'); - let exitCode: number | undefined; + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -89,21 +82,20 @@ test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS i AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); await vi.advanceTimersByTimeAsync(200); - assert.equal(exitCode, undefined); - const shutdownPromise = runtime?.shutdown(); + assert.equal(closed, false); + const shutdownPromise = runtime.close(); await vi.runOnlyPendingTimersAsync(); await shutdownPromise; - assert.equal(exitCode, 0); + await exited; + assert.equal(closed, true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -112,11 +104,7 @@ test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS i test('only a retained lease keeps the daemon runtime from self-reaping', async () => { vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-leases-'); - let exitCode: number | undefined; - let resolveExit: () => void; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -129,15 +117,12 @@ test('only a retained lease keeps the daemon runtime from self-reaping', async ( AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '80', }, }), - exit: (code) => { - exitCode = code; - resolveExit(); - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); const [registry] = leaseProbe.registries; registry!.allocateLease({ tenantId: 'tenant-a', runId: 'run-plain', ttlMs: 60_000 }); const retained = registry!.allocateLease({ @@ -148,13 +133,13 @@ test('only a retained lease keeps the daemon runtime from self-reaping', async ( }); await vi.advanceTimersByTimeAsync(80); - assert.equal(exitCode, undefined); + assert.equal(closed, false); registry!.releaseLease({ leaseId: retained.leaseId }); await vi.advanceTimersByTimeAsync(80); await vi.runOnlyPendingTimersAsync(); await exited; - assert.equal(exitCode, 0); + assert.equal(closed, true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts new file mode 100644 index 0000000000..bbf7bcfddf --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts @@ -0,0 +1,106 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { interactorResolution } from '../../core/interactor-resolution.ts'; +import { isActiveProviderDevice } from '../../core/provider-device-admission.ts'; +import { trackDownloadableArtifact } from '../../resources/artifact-tracking.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +async function startIsolatedRuntime(label: string, interactorTag: string, providerActive: boolean) { + const stateDir = mkdtempForTestSync(`agent-device-daemon-isolation-${label}-`); + const runtime = await startDaemonRuntime({ + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }), + stderr: { write: () => {} }, + stdout: { write: () => {} }, + composeDeps: daemonRuntimeDepsFixture({ + interactorResolution: Object.freeze({ + resolve: async () => { + throw new Error(interactorTag); + }, + }), + providerDeviceAdmission: Object.freeze({ isActive: () => providerActive }), + }), + }); + const { httpPort, token } = JSON.parse( + fs.readFileSync(resolveDaemonPaths(stateDir).infoPath, 'utf8'), + ) as { httpPort: number; token: string }; + return { runtime, stateDir, httpPort, token }; +} + +async function callSessionList(httpPort: number, token: string, requestId: string) { + const response = await fetch(`http://127.0.0.1:${httpPort}/rpc`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method: 'agent_device.command', + params: { command: 'session_list', meta: { requestId } }, + }), + }); + return (await response.json()) as { result?: unknown; error?: unknown }; +} + +test('two runtimes in one process serve, close and keep their state independently', async () => { + let a: Awaited> | undefined; + let b: Awaited> | undefined; + try { + a = await startIsolatedRuntime('a', 'interactor-a', true); + b = await startIsolatedRuntime('b', 'interactor-b', false); + expect(a.httpPort).not.toBe(b.httpPort); + expect(path.basename(a.stateDir)).not.toBe(path.basename(b.stateDir)); + + // Each runtime answers requests with its own token and its own state dir. + expect((await callSessionList(a.httpPort, a.token, 'iso-a-1')).error).toBeUndefined(); + expect((await callSessionList(b.httpPort, b.token, 'iso-b-1')).error).toBeUndefined(); + + // Per-runtime state: what one installs or tracks is invisible to the other. + expect(a.runtime.run(() => isActiveProviderDevice({} as never))).toBe(true); + expect(b.runtime.run(() => isActiveProviderDevice({} as never))).toBe(false); + await expect( + a.runtime.run(() => interactorResolution().resolve({} as never, {})), + ).rejects.toThrow('interactor-a'); + await expect( + b.runtime.run(() => interactorResolution().resolve({} as never, {})), + ).rejects.toThrow('interactor-b'); + + const artifactPath = path.join(a.stateDir, 'artifact.txt'); + fs.writeFileSync(artifactPath, 'artifact-a'); + const artifactId = a.runtime.run(() => + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'artifact.txt', + }), + ); + const fromB = await fetch( + `http://127.0.0.1:${b.httpPort}/artifacts/${encodeURIComponent(artifactId)}`, + { headers: { authorization: `Bearer ${b.token}`, connection: 'close' } }, + ); + expect(fromB.status).not.toBe(200); + await fromB.text(); + + // Closing one runtime leaves the other serving. + const reportA = await a.runtime.close(); + expect(reportA.sessions).toEqual({ closed: 0, failed: 0 }); + await expect(callSessionList(a.httpPort, a.token, 'iso-a-2')).rejects.toThrow(); + expect((await callSessionList(b.httpPort, b.token, 'iso-b-2')).error).toBeUndefined(); + expect(b.runtime.run(() => isActiveProviderDevice({} as never))).toBe(false); + + const reportB = await b.runtime.close(); + expect(reportB.plugins).toEqual({ disposed: [], failed: [] }); + } finally { + await a?.runtime.close(); + await b?.runtime.close(); + } +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts index 3d2892f5a8..e059b1d2f8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts @@ -63,8 +63,6 @@ test('an expired lease ends its session without any further request', async () = AGENT_DEVICE_LEASE_MIN_TTL_MS: '1', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -78,7 +76,7 @@ test('an expired lease ends its session without any further request', async () = await vi.advanceTimersByTimeAsync(1_000); assert.deepEqual(leaseProbe.ended, [[lease.leaseId, 'LEASE_EXPIRED']]); - const shutdown = runtime?.shutdown(); + const shutdown = runtime?.close(); await vi.runOnlyPendingTimersAsync(); await shutdown; } finally { @@ -106,14 +104,12 @@ test('shutdown cancels and drains lease ends before it tears sessions down', asy AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); assert.notEqual(runtime, null); - await runtime?.shutdown(); + await runtime?.close(); assert.deepEqual(leaseProbe.shutdownSteps, ['cancel', 'drain', 'teardown']); } finally { diff --git a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts index 7b506c5324..cc162432c2 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts @@ -4,6 +4,7 @@ import { mkdtempForTestSync, createTestHostEnvironment, } from '@agent-device/test-kit'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import fs from 'node:fs'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; @@ -100,7 +101,7 @@ import { daemonRuntimeDepsFixture, } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; -import { startDaemonRuntime, teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; import { makeIosSession } from '../../__tests__/support/session-factories.ts'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; @@ -116,7 +117,6 @@ afterEach(() => { test('daemon shutdown detaches before session teardown and force-finalizes only after gateway resources', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-shutdown-'); - const exits: number[] = []; const startupErrors: string[] = []; try { const runtime = await startDaemonRuntime({ @@ -135,9 +135,11 @@ test('daemon shutdown detaches before session teardown and force-finalizes only phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -153,17 +155,14 @@ test('daemon shutdown detaches before session teardown and force-finalizes only AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - registerProcessHandlers: false, stderr: { write: (chunk) => startupErrors.push(chunk) }, stdout: { write: () => {} }, }); expect(runtime, startupErrors.join('')).not.toBeNull(); - await Promise.all([runtime?.shutdown(), runtime?.shutdown()]); + await Promise.all([runtime?.close(), runtime?.close()]); expect(lifecycleEvents).toEqual(['detach', 'gateway-shutdown', 'finalize']); - expect(exits).toEqual([0]); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -190,9 +189,11 @@ test('a SIGTERM shutdown gives the handoff a diagnostics scope to write its reas phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -208,14 +209,12 @@ test('a SIGTERM shutdown gives the handoff a diagnostics scope to write its reas AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); expect(runtime).not.toBeNull(); - await runtime?.shutdown(); + await runtime?.close(); const daemonLog = fs.readFileSync(path.join(stateDir, 'daemon.log'), 'utf8'); expect(daemonLog).toMatch(/"phase":"detach_scope_probe"/); @@ -245,9 +244,11 @@ test('shutdown includes drain publications, releases their claims and refuses po phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -263,8 +264,6 @@ test('shutdown includes drain publications, releases their claims and refuses po AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -298,14 +297,14 @@ test('shutdown includes drain publications, releases their claims and refuses po refusal = error; } }); - await runtime!.shutdown(); + await runtime!.close(); expect(shutdownProbe.finalize).toHaveBeenCalledOnce(); expect(refusal).toMatchObject({ details: { reason: 'daemon_shutting_down' } }); expect(store.lookup('draining')).toBeUndefined(); expect(store.lookup('late')).toBeUndefined(); expect(fs.existsSync(deviceClaimFilePath(acquired.ownership.deviceKey))).toBe(false); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -327,9 +326,11 @@ test('shutdown joins dispatch completion after force-closing the client before t phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -345,8 +346,6 @@ test('shutdown joins dispatch completion after force-closing the client before t AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -363,7 +362,7 @@ test('shutdown joins dispatch completion after force-closing the client before t closed = resolve; }); let request: Promise | undefined; - let shutdown: Promise | undefined; + let shutdown: Promise | undefined; try { expect(runtime).not.toBeNull(); const store = shutdownProbe.store!; @@ -413,7 +412,7 @@ test('shutdown joins dispatch completion after force-closing the client before t shutdownProbe.drain = async () => { closed(); }; - shutdown = runtime!.shutdown(); + shutdown = runtime!.close(); await serversClosed; await new Promise((resolve) => setImmediate(resolve)); expect(lifecycleEvents).not.toContain('detach'); @@ -429,7 +428,7 @@ test('shutdown joins dispatch completion after force-closing the client before t release(); await request; await shutdown; - await runtime?.shutdown(); + await runtime?.close(); } }); test.each([false, true])( @@ -496,9 +495,11 @@ test('daemon shutdown releases a session lease whose session teardown rejects', phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -514,8 +515,6 @@ test('daemon shutdown releases a session lease whose session teardown rejects', AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -531,7 +530,7 @@ test('daemon shutdown releases a session lease whose session teardown rejects', }); shutdownProbe.finalize.mockRejectedValueOnce(new Error('teardown failed')); - await runtime?.shutdown(); + await runtime?.close(); expect(shutdownProbe.finalize).toHaveBeenCalledOnce(); expect(leaseProbe.released).toEqual([lease]); @@ -557,9 +556,11 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -575,8 +576,6 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -589,7 +588,7 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy retainOnClose: true, }); - await runtime?.shutdown(); + await runtime?.close(); expect(leaseProbe.released).toEqual([lease]); expect(leaseRegistry!.listActiveLeases()).toEqual([]); diff --git a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts index a873610674..d1c1592d47 100644 --- a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts @@ -28,7 +28,7 @@ type DaemonPaths = ReturnType; // anything that verifies identity against the process table. const SUCCESSOR_PID = 999_999_999; -function startRuntime(stateDir: string, exit: (code: number) => void) { +function startRuntime(stateDir: string) { return startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(() => ({ // The post-lock, pre-publication step the runtime awaits first. Making it throw lands the @@ -56,8 +56,6 @@ function startRuntime(stateDir: string, exit: (code: number) => void) { AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -90,15 +88,13 @@ test('a shutdown whose daemon.json names a successor keeps the file and logs the // daemon's metadata and leaving every later client failing with hasInfo:false + retainedLockProcess. const stateDir = mkdtempForTestSync('agent-device-daemon-info-ownership-shutdown-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); publishSuccessor(paths); - await runtime?.shutdown(); + await runtime?.close(); - expect(exits).toEqual([0]); expect(lifecycleEvents).toContain('gateway-shutdown'); expect(fs.existsSync(paths.infoPath), 'the serving daemon keeps its metadata').toBe(true); expect(fs.existsSync(paths.lockPath), 'the exiting daemon still releases its own lock').toBe( @@ -119,7 +115,7 @@ test('a shutdown removes its own metadata and reports an unverified release', as const stateDir = mkdtempForTestSync('agent-device-daemon-info-owned-shutdown-'); const paths = resolveDaemonPaths(stateDir); try { - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); const originalRmdir = fs.rmdirSync; @@ -127,7 +123,7 @@ test('a shutdown removes its own metadata and reports an unverified release', as if (target === paths.lockPath) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); return originalRmdir(target, options); }); - await runtime?.shutdown(); + await runtime?.close(); expect(fs.existsSync(paths.infoPath)).toBe(false); expect(logEvents(stateDir).map((event) => event.phase)).not.toContain( @@ -159,12 +155,12 @@ test('a skipped provider runtime is logged after this daemon publishes its recor code: 'INVALID_ARGS', }); providerComposition.skipped = [{ provider: 'limrun', error }]; - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); try { const skipped = logEvents(stateDir).find(({ phase }) => phase === 'provider_runtime_skipped'); expect(skipped?.data).toMatchObject({ provider: 'limrun', code: 'INVALID_ARGS' }); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -173,13 +169,9 @@ test('a startup that fails before publication removes no record and writes no lo // silent: reporting an "absent" record here would create a daemon.log on every such startup. const stateDir = mkdtempForTestSync('agent-device-daemon-info-startup-absent-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; startupFailure.active = true; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); - - expect(runtime).toBeNull(); - expect(exits).toEqual([1]); + await expect(startRuntime(stateDir)).rejects.toThrow('startup step failed'); expect(fs.existsSync(paths.infoPath)).toBe(false); expect(fs.existsSync(paths.logPath), 'nothing was declined, so nothing is logged').toBe(false); } finally { @@ -190,14 +182,10 @@ test('a startup that fails before publication removes no record and writes no lo test('a startup that fails before publication leaves a foreign record standing', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-info-startup-foreign-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; publishSuccessor(paths); startupFailure.active = true; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); - - expect(runtime).toBeNull(); - expect(exits).toEqual([1]); + await expect(startRuntime(stateDir)).rejects.toThrow('startup step failed'); expect(fs.existsSync(paths.infoPath)).toBe(true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); @@ -213,7 +201,7 @@ test('a successor published during this daemon lifetime is recorded at detection const stateDir = mkdtempForTestSync('agent-device-daemon-metadata-lost-'); const paths = resolveDaemonPaths(stateDir); try { - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); publishSuccessor(paths); @@ -227,7 +215,7 @@ test('a successor published during this daemon lifetime is recorded at detection ); vi.useRealTimers(); - await runtime?.shutdown(); + await runtime?.close(); } finally { vi.useRealTimers(); fs.rmSync(stateDir, { recursive: true, force: true }); @@ -249,13 +237,14 @@ test('both exits tear the watch down before they touch daemon.json', () => { // A watch still armed when the removal runs can report a loss for a record its own process just // deleted. The startup-failure half of this is unreachable from a test — everything after // publication needs a real toolchain — so the invariant is read off the source, the same way the - // arming order above is. + // arming order above is. Startup rollback runs its steps in reverse, so the registration that + // stops the watch comes after the one that finishes the registration; the close path stops first. const source = fs.readFileSync(new URL('./daemon-runtime.ts', import.meta.url), 'utf8'); const stops = [...source.matchAll(/stopMetadataLossWatch\(\);/g)].map((m) => m.index); const finishes = [...source.matchAll(/await finishDaemonRegistration\(/g)].map((m) => m.index); expect(stops).toHaveLength(2); expect(finishes).toHaveLength(2); - expect(finishes[0]).toBeGreaterThan(stops[0]!); - expect(finishes[0]).toBeLessThan(stops[1]!); + expect(stops[0]).toBeGreaterThan(finishes[0]!); + expect(stops[1]).toBeGreaterThan(stops[0]!); expect(finishes[1]).toBeGreaterThan(stops[1]!); }); diff --git a/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts b/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts index a838601426..21e1c4f71e 100644 --- a/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts @@ -33,8 +33,6 @@ function startDaemon( AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ @@ -70,7 +68,7 @@ test('daemon startup configures its platform owners before it publishes readines }), ]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -94,6 +92,6 @@ test('a platform owner diagnostic about a refused restore is logged and the daem }), ]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts b/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts index 8304b4c083..ed04ed819e 100644 --- a/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts @@ -35,8 +35,6 @@ test('daemon startup reaps orphaned simctl recorders with a graceful finalize wi AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -50,7 +48,7 @@ test('daemon startup reaps orphaned simctl recorders with a graceful finalize wi // so it releases the host recording lock instead of leaving it dangling for the next session. expect(Number(recordingReap!.termTimeoutMs)).toBeGreaterThanOrEqual(5_000); - await runtime?.shutdown(); + await runtime?.close(); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts index f8b95f5e83..1e790f0321 100644 --- a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts @@ -1,11 +1,11 @@ import path from 'node:path'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import { afterEach, expect, test, vi } from 'vitest'; import type { ScreenRecordingCompletion } from '@agent-device/contracts/screen-recording-runtime'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { SessionStore } from '../../core/session-store.ts'; import { makeRecordingSession } from '../../__tests__/support/session-teardown.fixtures.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts b/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts index ec814ad7e9..1526f796b8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts @@ -52,7 +52,6 @@ test('a /rpc flow failure the server refused itself reaches the runtime stderr a expect(observed.events).toHaveLength(1); expect(observed.events[0]?.data).toEqual({ code: 'UNKNOWN', message: 'Unknown error' }); expect(observed.rejections).toEqual([]); - expect(observed.exitCode).toBeUndefined(); expect(observed.nextCall).toEqual({ status: 400, code: -32600 }); }); @@ -67,7 +66,6 @@ test('a /rpc flow failure still reaches daemon.log when the injected stderr thro expect(observed.events[0]?.phase).toBe('daemon_rpc_request_flow_failed'); expect(observed.stderrLines).toEqual([]); expect(observed.rejections).toEqual([]); - expect(observed.exitCode).toBeUndefined(); }); type FlowFailure = { @@ -76,7 +74,6 @@ type FlowFailure = { events: Array<{ phase: string; data?: Record }>; stderrLines: string[]; rejections: unknown[]; - exitCode: number | undefined; }; /** @@ -94,8 +91,7 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise { rejections.push(reason); }; - let exitCode: number | undefined; - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(), @@ -107,10 +103,6 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: (chunk: string) => { if (options.stderrThrows) throw new Error('the stream is gone'); @@ -143,14 +135,13 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise ({ @@ -77,7 +77,7 @@ for (const initialSession of ['published', 'draft'] as const) { (error: unknown) => ({ error }), ); const stderr: string[] = []; - let teardown: Promise | undefined; + let teardown: Promise | undefined; try { await captureEntered.promise; vi.useFakeTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts index 46e2b2d332..12376f23d9 100644 --- a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts @@ -1,10 +1,10 @@ import path from 'node:path'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import { afterEach, expect, test, vi } from 'vitest'; import { mkdtempForTestSync, WEB_DESKTOP_DEVICE } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { makeWebSession } from '../../__tests__/support/session-teardown.fixtures.ts'; import { SessionStore } from '../../core/session-store.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 4d8643be27..2102753bec 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -35,14 +35,23 @@ import { getLeaseRegistryExecutionLocks, } from '../request-execution-scope.ts'; import type { LeaseEndReason } from '../../sessions/lease-lifecycle.ts'; -import { stopSessionAppLog, teardownSessionResources } from '../../sessions/session-teardown.ts'; +import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; import { finalizeDaemonSessionApplicationLifecycle } from '../application-lifecycle-recovery.ts'; import { runtimeHintValues } from '../../sessions/session-runtime.ts'; -import { closeDaemonServers } from './server-shutdown.ts'; +import { + closeDaemonServers, + createRuntimeContext, + teardownDaemonSessionForShutdown, + type RuntimeContext, + type WritableOutput, + createProviderRuntimeDisposal, + createStartupRollback, + type ShutdownReport, + type StartupRollback, +} from './server-shutdown.ts'; import type { DaemonInvokeFn } from '../../core/daemon-request.ts'; -import type { SessionRef, SessionState } from '../../core/session-state.ts'; -import type { RuntimeHintValues } from '@agent-device/contracts/application-lifecycle-runtime'; +import type { SessionRef } from '../../core/session-state.ts'; import { createDaemonIdleReap } from '../../sessions/server/daemon-idle-reap.ts'; import { withDaemonDiagnosticsScope } from '@agent-device/daemon-contracts/daemon-diagnostics-scope'; import { createSessionIdleExpiry } from './daemon-session-idle-expiry.ts'; @@ -58,6 +67,7 @@ import { createAudioProbeAdmissionLedger } from '@agent-device/capture-kit/audio import { createPerfCaptureAdmissionLedger } from '@agent-device/capture-kit/perf-capture-admission-ledger'; import { createScreenRecordingAdmissionLedger } from '@agent-device/capture-kit/screen-recording-admission-ledger'; import { withCommandHost } from '@agent-device/host-kit/command'; +import { type RuntimeStateHandle } from '@agent-device/host-kit/request'; import { emitDiagnostic, type ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; import { @@ -85,12 +95,11 @@ import { listenNetServer, type DaemonServer, } from './transport.ts'; -import { prewarmPngWorker, terminatePngWorker } from '@agent-device/capture-kit/png-worker-client'; +import { retainPngWorker } from '@agent-device/capture-kit/png-worker-client'; import { openWebSessionNames } from '../../sessions/web-session-names.ts'; import { createAppLogAdmissionLedger } from '../../resources/app-log-admission-ledger.ts'; import type { AppLogSessionArtifacts } from '@agent-device/contracts/app-log-runtime'; import type { PlatformOwnerLifecycle } from '../../core/platform-owner-lifecycle.ts'; -import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts'; import type { LeaseLifecycleProvider, ProviderDeviceRuntime, @@ -107,10 +116,6 @@ const DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS = 2_000; // (#2170). Bound the grace to the recorder purpose, so daemon startup stays under the client budget. const DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS = 5_000; -type WritableOutput = { - write: (chunk: string) => unknown; -}; - /** * Every capability the daemon runtime cannot construct for itself: the platform and provider * runtimes, host diagnostics, credential fingerprinting, resource cleanup, the platform lifecycle @@ -121,6 +126,8 @@ type WritableOutput = { * same factory, so the production assembly is the one every runtime test boots. */ export type DaemonRuntimeDeps = Readonly<{ + /** State the composition's platform modules keep per daemon runtime (the Apple runner's). */ + runtimeStates: readonly RuntimeStateHandle[]; ownedProcessRecords: OwnedProcessRecordStore; hostDiagnostics: HostDiagnostics; ownerLifecycle: PlatformOwnerLifecycle; @@ -198,8 +205,6 @@ export type DaemonRuntimeOptions = { host: HostEnvironment; stdout?: WritableOutput; stderr?: WritableOutput; - exit?: (code: number) => void; - registerProcessHandlers?: boolean; /** The root's runtime composition, as a factory so the server owns paths, policy, and store. */ composeDeps: ComposeDaemonRuntimeDeps; }; @@ -207,10 +212,27 @@ export type DaemonRuntimeOptions = { export type DaemonRuntimeController = { httpPort?: number; socketPort?: number; - shutdown: (options?: { exitCode?: number; cause?: unknown }) => Promise; + /** + * Closes the runtime. Concurrent and repeated calls share one close, whose report every caller + * receives. The runtime never exits the process: the executable owns signals and exit codes. + */ + close: (options?: { cause?: unknown }) => Promise; + /** Settles once the runtime has closed, whoever closed it (an idle reap closes it unasked). */ + closed: Promise; + /** Runs `fn` as this runtime, for code that serves it from outside its own callbacks. */ + run: (fn: () => T) => T; token: string; }; +/** + * Thrown by {@link startDaemonRuntime} when the daemon must not run: its policy is unusable or + * another daemon owns the state dir. `details.exitCode` is the code the executable exits with. + */ +export function daemonStartupExitCode(error: unknown): number { + const exitCode = asAppError(error).details?.exitCode; + return typeof exitCode === 'number' ? exitCode : 1; +} + export async function flushDaemonStartupDiagnostics( logPath: string, diagnostics: readonly ResourceDiagnostic[], @@ -288,126 +310,34 @@ function armDaemonMetadataLossWatch( }); } -async function settleDaemonTeardownStep(params: { - session: SessionState; - stderr: WritableOutput; - resource: 'app-log' | 'session' | 'lifecycle'; - teardown: () => Promise; -}): Promise { - const { session, stderr, resource, teardown } = params; - try { - await teardown(); - return true; - } catch (error) { - stderr.write( - `Daemon ${resource} teardown error (${session.name}): ${ - error instanceof Error ? error.message : String(error) - }\n`, - ); - return false; - } -} - -/** - * Daemon-shutdown teardown of one session: bounded resource cleanup (budget - * from {@link resolveDaemonSessionTeardownTimeoutMs}, resolved BEFORE cleanup - * starts since finalizing the recording detaches `session.screenRecording`), then the - * repair-commit finalization and session deletion. Cleanup failures — including - * a recorder that could not be finalized — surface on stderr instead of being - * silently swallowed. - */ -export async function teardownDaemonSessionForShutdown(params: { - ref: SessionRef; - sessionStore: SessionStore; - stderr: WritableOutput; - platformCleanup: PlatformResourceCleanup; - finalizeApplicationLifecycle?: ( - session: SessionState, - runtimeHints: RuntimeHintValues, - ) => Promise; - afterSuccessfulTeardown?: (session: SessionState) => Promise; -}): Promise { - const { - ref, - sessionStore, - stderr, - platformCleanup, - finalizeApplicationLifecycle, - afterSuccessfulTeardown, - } = params; - const current = sessionStore.resolveCurrent(ref); - const session = current ?? ref.session; - const runtimeHints = runtimeHintValues( - current ? sessionStore.getRuntimeHints(ref.address) : undefined, - ); - const timeoutMs = resolveDaemonSessionTeardownTimeoutMs(session); - // The ownership-fenced app-log side effect must settle while this process - // still owns the daemon lock. It is intentionally outside the generic - // teardown race so lock release and runtime shutdown cannot overtake it. - - const appLogTeardownSucceeded = await settleDaemonTeardownStep({ - session, - stderr, - resource: 'app-log', - teardown: async () => await stopSessionAppLog({ ref, sessionStore }), - }); - const sessionAfterAppLog = sessionStore.resolveCurrent(ref) ?? session; - const teardown = (async () => { - const genericTeardownSucceeded = await settleDaemonTeardownStep({ - session, - stderr, - resource: 'session', - teardown: async () => - await teardownSessionResources({ - appLog: 'already-settled', - ref, - sessionStore, - platformCleanup, - }), - }); - const lifecycleTeardownSucceeded = finalizeApplicationLifecycle - ? await settleDaemonTeardownStep({ - session, - stderr, - resource: 'lifecycle', - teardown: async () => - await finalizeApplicationLifecycle(sessionAfterAppLog, runtimeHints), - }) - : true; - return genericTeardownSucceeded && lifecycleTeardownSucceeded; - })(); - const genericTeardownSucceeded = await Promise.race([ - teardown, - sleep(timeoutMs).then(() => { - stderr.write(`Daemon session teardown timed out (${session.name}).\n`); - return false; - }), - ]); - const teardownSucceeded = appLogTeardownSucceeded && genericTeardownSucceeded; - // ADR 0012 decision 6, R7 + commit semantics (C2/C5a): commit the healed - // `.ad` iff the repair transaction completed, else leave a bounded - // `REPAIR_SESSION_EXPIRED` tombstone for the reaped-before-finalize case. - sessionStore.finalizeRepairTeardown(ref); - if (teardownSucceeded) await afterSuccessfulTeardown?.(session); - sessionStore.retire(ref); -} - export async function startDaemonRuntime( options: DaemonRuntimeOptions, -): Promise { - return await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { - return await startDaemonRuntimeOnHost(options); - }); +): Promise { + const runtimeContext = createRuntimeContext(); + return await runtimeContext.run( + async () => + await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { + const rollback = createStartupRollback(); + rollback.onRollback(() => runtimeContext.dispose()); + try { + return await startDaemonRuntimeOnHost(options, rollback, runtimeContext); + } catch (error) { + await rollback.run(); + throw error; + } + }), + ); } async function startDaemonRuntimeOnHost( options: DaemonRuntimeOptions, -): Promise { + rollback: StartupRollback, + runtimeContext: RuntimeContext, +): Promise { const { host } = options; const { env } = host; const stdout = options.stdout ?? process.stdout; const stderr = options.stderr ?? process.stderr; - const exit = options.exit ?? ((code: number) => process.exit(code)); const daemonPaths = resolveDaemonPaths(env.AGENT_DEVICE_STATE_DIR, { env, cwd: host.cwd }); const { baseDir, infoPath, logPath, sessionsDir } = daemonPaths; const daemonServerMode = resolveDaemonServerMode(env.AGENT_DEVICE_DAEMON_SERVER_MODE); @@ -415,14 +345,7 @@ async function startDaemonRuntimeOnHost( // ADR 0029: a policy that cannot be read or validated stops startup; the daemon never runs // with a weaker policy than its operator named. The server loads it before composing any // runtime, so no capability is ever built under a policy that failed validation. - let daemonPolicy: DaemonPolicy | undefined; - try { - daemonPolicy = loadDaemonPolicy(env, host.cwd); - } catch (error) { - stderr.write(`Daemon error: ${asAppError(error).message}\n`); - exit(1); - return null; - } + const daemonPolicy: DaemonPolicy | undefined = loadDaemonPolicy(env, host.cwd); // Derived here, beside the policy it reads: the server hands the same refusal to the root's // composition, so a runtime can never be built without the operator's shutdown gate. const assertShutdownAllowed = daemonPolicy @@ -442,6 +365,7 @@ async function startDaemonRuntimeOnHost( }), openWebSessionNames: () => openWebSessionNames(sessionStore), }); + runtimeContext.extend(deps.runtimeStates); const appLogAdmissionLedger = createAppLogAdmissionLedger(); const audioProbeAdmissionLedger = createAudioProbeAdmissionLedger(); const perfCaptureAdmissionLedger = createPerfCaptureAdmissionLedger(); @@ -452,6 +376,10 @@ async function startDaemonRuntimeOnHost( const daemonCodeOrigin = resolveDaemonCodeOrigin(); const daemonCodeSignature = resolveDaemonCodeSignature(); const providerDeviceRuntimes = deps.providerDeviceRuntimes; + const pluginDisposal = createProviderRuntimeDisposal(providerDeviceRuntimes); + rollback.onRollback(async () => { + await pluginDisposal.dispose(); + }); const deviceRuntimeGateway = deps.deviceRuntimeGateway; const applicationLifecycle = deviceRuntimeGateway.applicationLifecycle; if (!applicationLifecycle) { @@ -468,6 +396,7 @@ async function startDaemonRuntimeOnHost( stateDir: baseDir, recoverableProviderIds: deps.recoverableProviderIds, }); + rollback.onRollback(() => expiredProviderLeaseReleaser.shutdown()); void expiredProviderLeaseReleaser.retryPending(); const leaseRegistry = new LeaseRegistry({ maxActiveSimulatorLeases: parseIntegerEnv(env.AGENT_DEVICE_MAX_SIMULATOR_LEASES), @@ -486,6 +415,7 @@ async function startDaemonRuntimeOnHost( LEASE_EXPIRY_SWEEP_INTERVAL_MS, ); leaseExpirySweep.unref(); + rollback.onRollback(() => clearInterval(leaseExpirySweep)); const dispatchRequest = createRequestHandler({ host, logPath, @@ -525,10 +455,10 @@ async function startDaemonRuntimeOnHost( const shutdownClaimLedger = createDaemonShutdownClaimLedger(); - const teardownDaemonSession = async (ref: SessionRef): Promise => { + const teardownDaemonSession = async (ref: SessionRef): Promise => { const session = sessionStore.resolveCurrent(ref) ?? ref.session; try { - await teardownDaemonSessionForShutdown({ + return await teardownDaemonSessionForShutdown({ ref, sessionStore, stderr, @@ -548,10 +478,14 @@ async function startDaemonRuntimeOnHost( } }; - const teardownDaemonSessions = async (): Promise => { + const teardownDaemonSessions = async (): Promise => { sessionStore.closeAdmission(); const sessionsToStop = sessionStore.listRefs(); - await Promise.all(sessionsToStop.map(teardownDaemonSession)); + const outcomes = await Promise.all( + sessionsToStop.map(async (ref) => await teardownDaemonSession(ref).catch(() => false)), + ); + const closedCount = outcomes.filter(Boolean).length; + return { closed: closedCount, failed: outcomes.length - closedCount }; }; // #2833: settles the resources of a session this daemon expires for idleness. Deliberately NOT @@ -621,7 +555,7 @@ async function startDaemonRuntimeOnHost( // Reaps this daemon process when it sits fully idle (no open sessions, no // in-flight requests, no active recording) past AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS. - // `shutdown` is defined below but only invoked asynchronously by the timer, + // `close` is defined below but only invoked asynchronously by the timer, // well after this closure captures it. let shuttingDown = false; const inFlightRequests = new Set>(); @@ -630,11 +564,17 @@ async function startDaemonRuntimeOnHost( getInFlightRequestCount: () => inFlightRequests.size, hasRetainedLeases: () => leaseRegistry.hasRetainedLeases(), onIdleReap: () => { - void shutdown(); + void close(); }, env, }); + rollback.onRollback(() => { + idleReap.cancel(); + sessionIdleExpiry.cancel(); + leaseSessionEnd.cancel(); + }); + const handleRequest: DaemonInvokeFn = async (req) => { if (shuttingDown) return { @@ -646,7 +586,7 @@ async function startDaemonRuntimeOnHost( }, }; idleReap.cancel(); - const dispatch = dispatchRequest(req); + const dispatch = runtimeContext.run(() => dispatchRequest(req)); inFlightRequests.add(dispatch); try { return await dispatch; @@ -741,129 +681,121 @@ async function startDaemonRuntimeOnHost( const acquisition = await tryAcquireDaemonRegistration(daemonPaths); if (acquisition.status !== 'acquired') { - await Promise.allSettled( - providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), - ); - stderr.write(`Daemon registration ${acquisition.status}; exiting.\n`); - exit(DAEMON_STARTUP_EXIT_CODES[acquisition.status]); - return null; + throw new AppError('COMMAND_FAILED', `Daemon registration ${acquisition.status}.`, { + reason: 'daemon_registration_refused', + status: acquisition.status, + exitCode: DAEMON_STARTUP_EXIT_CODES[acquisition.status], + }); } const registration = acquisition.owner; - + rollback.onRollback(async () => { + await deps.ownerLifecycle.clearDaemonLockConfiguration(); + }); + rollback.onRollback(async () => { + await finishDaemonRegistration({ registration, infoPath, logPath }); + }); let servers: DaemonServer[] = []; - let socketPort: number | undefined; - let httpPort: number | undefined; + rollback.onRollback(async () => { + closeServersBestEffort(servers); + stopMetadataLossWatch(); + }); const startupDiagnostics: ResourceDiagnostic[] = []; - try { - await deps.ownerLifecycle.configureForDaemonLock({ - stateDir: baseDir, - hasDeviceClaimAuthority: (device) => processOwnsActiveDeviceClaim(device, env), - onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), - }); - const legacyMarkerRecovery = await deps.ownerLifecycle.recoverLegacyAppLogMarkers(sessionsDir); - appLogAdmissionLedger.retainLegacyMarkers(legacyMarkerRecovery.retained); - for (const markerPath of legacyMarkerRecovery.recovered) { - startupDiagnostics.push({ - phase: 'app_log_legacy_marker_recovered', - resourcePath: markerPath, - data: {}, - }); - } - for (const retained of legacyMarkerRecovery.retained) { - startupDiagnostics.push({ - phase: 'app_log_legacy_marker_retained', - resourcePath: retained.markerPath, - data: { - reason: retained.reason, - ...(retained.message === undefined ? {} : { message: retained.message }), - }, - }); - } - await recoverAppLogResourcesAfterDaemonLock({ - sessionsDir, - gateway: deviceRuntimeGateway, - scope: createRecoveryPlatformScope(), - onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + await deps.ownerLifecycle.configureForDaemonLock({ + stateDir: baseDir, + hasDeviceClaimAuthority: (device) => processOwnsActiveDeviceClaim(device, env), + onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + }); + const legacyMarkerRecovery = await deps.ownerLifecycle.recoverLegacyAppLogMarkers(sessionsDir); + appLogAdmissionLedger.retainLegacyMarkers(legacyMarkerRecovery.retained); + for (const markerPath of legacyMarkerRecovery.recovered) { + startupDiagnostics.push({ + phase: 'app_log_legacy_marker_recovered', + resourcePath: markerPath, + data: {}, }); - await reapOwnedProcessRecordsAtStartup(deps.ownedProcessRecords, { - openWebSessionNames: openWebSessionNames(sessionStore), - purposes: ['simctl-screen-recording'], - termTimeoutMs: DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS, + } + for (const retained of legacyMarkerRecovery.retained) { + startupDiagnostics.push({ + phase: 'app_log_legacy_marker_retained', + resourcePath: retained.markerPath, + data: { + reason: retained.reason, + ...(retained.message === undefined ? {} : { message: retained.message }), + }, }); - await deps.cleanupManagedWebOrphans({ - stateDir: baseDir, - openWebSessionNames: openWebSessionNames(sessionStore), - ownedProcessRecords: deps.ownedProcessRecords, + } + await recoverAppLogResourcesAfterDaemonLock({ + sessionsDir, + gateway: deviceRuntimeGateway, + scope: createRecoveryPlatformScope(), + onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + }); + await reapOwnedProcessRecordsAtStartup(deps.ownedProcessRecords, { + openWebSessionNames: openWebSessionNames(sessionStore), + purposes: ['simctl-screen-recording'], + termTimeoutMs: DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS, + }); + await deps.cleanupManagedWebOrphans({ + stateDir: baseDir, + openWebSessionNames: openWebSessionNames(sessionStore), + ownedProcessRecords: deps.ownedProcessRecords, + }); + // Marker-gated lifecycle recovery owns test-IME orphan repair. Its implementation remains + // lazy until the marker exists, so a normal daemon startup does not load or probe adb. + void applicationLifecycle.recoverStartupResources({ stateDir: baseDir }).catch((error) => { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_lifecycle_startup_recovery_failed', + data: { error: error instanceof Error ? error.message : String(error) }, }); - // Marker-gated lifecycle recovery owns test-IME orphan repair. Its implementation remains - // lazy until the marker exists, so a normal daemon startup does not load or probe adb. - void applicationLifecycle.recoverStartupResources({ stateDir: baseDir }).catch((error) => { - emitDiagnostic({ - level: 'warn', - phase: 'daemon_lifecycle_startup_recovery_failed', - data: { error: error instanceof Error ? error.message : String(error) }, - }); + }); + const opened = await openDaemonServers(); + servers = opened.servers; + const { socketPort, httpPort } = opened; + publishDaemonInfo(socketPort, httpPort); + stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); + await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); + for (const skipped of deps.skippedProviderRuntimes) { + await emitDaemonDiagnostic(logPath, 'provider_runtime_skipped', { + provider: skipped.provider, + code: skipped.error.code, + message: skipped.error.message, + hint: skipped.error.details?.hint, }); - const opened = await openDaemonServers(); - servers = opened.servers; - socketPort = opened.socketPort; - httpPort = opened.httpPort; - publishDaemonInfo(socketPort, httpPort); - stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); - await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); - for (const skipped of deps.skippedProviderRuntimes) { - await emitDaemonDiagnostic(logPath, 'provider_runtime_skipped', { - provider: skipped.provider, - code: skipped.error.code, - message: skipped.error.message, - hint: skipped.error.details?.hint, - }); - } - // After publication: publishDaemonInfo truncates daemon.log, so anything - // written before it is lost — including reconciliation diagnostics. - await reconcileDeviceClaimsForDaemonStartup( - logPath, - createOwnerScopedDeviceClaimReconciler({ - scope: createRecoveryPlatformScope(), - composeGateway: deps.claimRecoveryGateway, - }), - baseDir, - env, - ); - // Arms the initial idle-reap timer: a daemon that starts and never - // receives a request must still be able to reap itself. - idleReap.noteActivity(); - // The #2833 deadline needs no equivalent arm: the store is empty until a request puts a session - // in it, and that request's own completion re-arms the reaper. - } catch (error) { - const appErr = asAppError(error); - stderr.write(`Daemon error: ${appErr.message}\n`); - closeServersBestEffort(servers); - stopMetadataLossWatch(); - await Promise.allSettled( - providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), - ); - await finishDaemonRegistration({ registration, infoPath, logPath }); - await deps.ownerLifecycle.clearDaemonLockConfiguration(); - exit(1); - return null; } + // After publication: publishDaemonInfo truncates daemon.log, so anything + // written before it is lost — including reconciliation diagnostics. + await reconcileDeviceClaimsForDaemonStartup( + logPath, + createOwnerScopedDeviceClaimReconciler({ + scope: createRecoveryPlatformScope(), + composeGateway: deps.claimRecoveryGateway, + }), + baseDir, + env, + ); + // Arms the initial idle-reap timer: a daemon that starts and never + // receives a request must still be able to reap itself. + idleReap.noteActivity(); + // The #2833 deadline needs no equivalent arm: the store is empty until a request puts a session + // in it, and that request's own completion re-arms the reaper. // Spawn the PNG worker ahead of the first screenshot request so its // cold-start cost is not paid on a user-visible call. Best effort: when it // cannot start, PNG processing falls back to the in-process sync path. - prewarmPngWorker(); + const releasePngWorker = retainPngWorker(); - const shutdownOnHost = async (shutdownOptions: { exitCode?: number; cause?: unknown } = {}) => { + let registrationFinished = false; + let lockConfigurationCleared = false; + const closeSteps = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); sessionIdleExpiry.cancel(); leaseSessionEnd.cancel(); clearInterval(leaseExpirySweep); - if (shuttingDown) return; shuttingDown = true; stopMetadataLossWatch(); - if (shutdownOptions.cause) { - await emitFatalDiagnostic(shutdownOptions.cause); + if (closeOptions.cause) { + await emitFatalDiagnostic(closeOptions.cause); } await closeDaemonServers(servers); await Promise.allSettled(inFlightRequests); @@ -875,13 +807,14 @@ async function startDaemonRuntimeOnHost( // no diagnostics scope, so `emitDiagnostic` would drop every detach reason. Daemon debug level is // forced on here because the declines are the point of the record — a handoff that silently // skipped is indistinguishable from a rebuild (#2681). + let handedOffRunners: readonly string[] = []; try { await withDaemonDiagnosticsScope({ logPath }, async () => { - await applicationLifecycle.detachForDaemonShutdown(); + handedOffRunners = await applicationLifecycle.detachForDaemonShutdown(); }); } catch {} expiredProviderLeaseReleaser.beginShutdown(); - await teardownDaemonSessions(); + const sessions = await teardownDaemonSessions(); await withDaemonDiagnosticsScope({ logPath }, async () => { await finalizeDaemonLeases({ leaseRegistry, @@ -913,58 +846,83 @@ async function startDaemonRuntimeOnHost( }); expiredProviderLeaseReleaser.shutdown(); await deviceRuntimeGateway.shutdown(); - await Promise.allSettled( - providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), - ); - await applicationLifecycle.finalizeDaemonShutdown(); + const plugins = await pluginDisposal.dispose(); + const stoppedRunners = await applicationLifecycle.finalizeDaemonShutdown(); // Best effort: stop the PNG worker so an in-flight job cannot delay exit. await Promise.race([ - terminatePngWorker().catch(() => {}), + releasePngWorker().catch(() => {}), sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), ]); + registrationFinished = true; await finishDaemonRegistration({ registration, infoPath, logPath, outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, }); + lockConfigurationCleared = true; await deps.ownerLifecycle.clearDaemonLockConfiguration(); - exit(shutdownOptions.exitCode ?? 0); + const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); + return { + providerReleases: { + completed: providerReleaseDrain.released.map((lease) => lease.leaseId), + pending: providerReleaseDrain.pending + .filter((lease) => !unknownLeaseIds.has(lease.leaseId)) + .map((lease) => lease.leaseId), + unknown: [...unknownLeaseIds], + }, + runners: { stopped: [...stoppedRunners], handedOff: [...handedOffRunners] }, + sessions, + plugins, + }; }; - // Signal handlers and external callers run outside the startup scope, so shutdown re-enters it. - const shutdown = async (shutdownOptions: { exitCode?: number; cause?: unknown } = {}) => - await withCommandHost( - { host, requestEnv: undefined }, - async () => await shutdownOnHost(shutdownOptions), - ); + // A step that rejects must not strand what the later steps release: the PNG worker hold, the + // provider runtimes, the registration and lock, and the runtime's own state. + const closeOnHost = async (closeOptions: { cause?: unknown }): Promise => { + try { + return await closeSteps(closeOptions); + } finally { + await Promise.allSettled([ + releasePngWorker(), + pluginDisposal.dispose(), + registrationFinished + ? undefined + : finishDaemonRegistration({ registration, infoPath, logPath }), + lockConfigurationCleared ? undefined : deps.ownerLifecycle.clearDaemonLockConfiguration(), + ]); + runtimeContext.dispose(); + } + }; - if (options.registerProcessHandlers !== false) { - process.on('SIGINT', () => { - void shutdown(); - }); - process.on('SIGTERM', () => { - void shutdown(); - }); - process.on('SIGHUP', () => { - void shutdown(); - }); - process.on('uncaughtException', (err) => { - const appErr = err instanceof AppError ? err : asAppError(err); - stderr.write(`Daemon error: ${appErr.message}\n`); - void shutdown({ exitCode: 1, cause: err }); - }); - process.on('unhandledRejection', (reason) => { - const err = reason instanceof Error ? reason : new Error(String(reason)); - const appErr = err instanceof AppError ? err : asAppError(err); - stderr.write(`Daemon error: ${appErr.message}\n`); - void shutdown({ exitCode: 1, cause: err }); - }); - } + // Idle reap, external callers and the executable's signal handlers all close through here, and + // none of them runs inside the startup scope, so the close re-enters it. + let closing: Promise | undefined; + let settleClosed: { resolve(report: ShutdownReport): void; reject(error: unknown): void }; + const closed = new Promise((resolve, reject) => { + settleClosed = { resolve, reject }; + }); + // A close nobody awaits (an idle reap) still reports through `closed`, which must not become an + // unhandled rejection when no caller observes it. + closed.catch(() => {}); + const close = (closeOptions: { cause?: unknown } = {}): Promise => { + if (!closing) { + closing = runtimeContext.run(() => + withCommandHost( + { host, requestEnv: undefined }, + async () => await closeOnHost(closeOptions), + ), + ); + closing.then(settleClosed.resolve, settleClosed.reject); + } + return closing; + }; return { httpPort, - shutdown, + close, + closed, + run: runtimeContext.run, socketPort, token, }; diff --git a/packages/daemon/src/server/http/http-server.ts b/packages/daemon/src/server/http/http-server.ts index e5e5d878f6..6259b0df7d 100644 --- a/packages/daemon/src/server/http/http-server.ts +++ b/packages/daemon/src/server/http/http-server.ts @@ -6,6 +6,7 @@ // in `http-rpc-envelope.ts`, the authorization decisions in // `http-authorization.ts`, and the RPC method vocabulary in `http-rpc-methods.ts`. import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import { AsyncResource } from 'node:async_hooks'; import http from 'node:http'; import { randomUUID } from 'node:crypto'; import type { DiagnosticsRecordRef, NormalizedError } from '@agent-device/kernel/errors'; @@ -104,7 +105,7 @@ export async function createDaemonHttpServer(options: { resolveRequestDiagnosticsPath, onUnownedRequestFailure, } = options; - return http.createServer((req, res) => { + const onRequest = AsyncResource.bind((req: http.IncomingMessage, res: http.ServerResponse) => { if (req.method === 'GET' && req.url === '/health') { res.statusCode = 200; res.setHeader('content-type', 'application/json'); @@ -212,6 +213,7 @@ export async function createDaemonHttpServer(options: { }); }); }); + return http.createServer(onRequest); } /** Accumulates one `/rpc` body and hands it to the flow once the stream ends. */ @@ -235,9 +237,12 @@ function readRpcRequestBody( } }); - req.on('end', () => { - onBody(body); - }); + req.on( + 'end', + AsyncResource.bind(() => { + onBody(body); + }), + ); } // The `/rpc` flow, in the order it runs: the envelope decides whether this body is a @@ -471,8 +476,9 @@ function beginRpcRequestLifetime(params: { }; }, watchForClientDisconnect() { - req.on('aborted', markCanceledIfResponseIncomplete); - res.on('close', markCanceledIfResponseIncomplete); + const onClientGone = AsyncResource.bind(markCanceledIfResponseIncomplete); + req.on('aborted', onClientGone); + res.on('close', onClientGone); if (req.aborted || res.destroyed) { markCanceledIfResponseIncomplete(); } diff --git a/packages/daemon/src/server/http/server-shutdown.ts b/packages/daemon/src/server/http/server-shutdown.ts index b86f71a670..60249cdbf2 100644 --- a/packages/daemon/src/server/http/server-shutdown.ts +++ b/packages/daemon/src/server/http/server-shutdown.ts @@ -1,4 +1,23 @@ +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import type { DaemonServer } from './transport.ts'; +import { sleep } from '@agent-device/host-kit/retry'; +import { + createRequestCancelRuntimeState, + type RuntimeStateHandle, +} from '@agent-device/host-kit/request'; +import type { RuntimeHintValues } from '@agent-device/contracts/application-lifecycle-runtime'; +import { SessionStore } from '../../core/session-store.ts'; +import type { SessionRef, SessionState } from '../../core/session-state.ts'; +import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts'; +import { createProviderDeviceAdmissionRuntimeState } from '../../core/provider-device-admission.ts'; +import { createInteractorResolutionRuntimeState } from '../../core/interactor-resolution.ts'; +import { createArtifactTrackingRuntimeState } from '../../resources/artifact-tracking.ts'; +import { createResumableUploadRuntimeState } from '../resumable-upload.ts'; +import { createMaterializedPathRuntimeState } from '../../sessions/materialized-path-registry.ts'; +import { stopSessionAppLog, teardownSessionResources } from '../../sessions/session-teardown.ts'; +import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; +import { runtimeHintValues } from '../../sessions/session-runtime.ts'; const DAEMON_SHUTDOWN_TIMEOUT_MS = 5_000; @@ -37,3 +56,208 @@ export async function closeDaemonServers( }), ); } + +/** What a closed daemon runtime leaves behind, by kind of resource. */ +export type ShutdownReport = { + /** Provider lease releases, by lease id. `unknown` was still in flight when the budget ran out. */ + providerReleases: { completed: string[]; pending: string[]; unknown: string[] }; + /** Device runners, by device id: `handedOff` was left running for the next daemon. */ + runners: { stopped: string[]; handedOff: string[] }; + sessions: { closed: number; failed: number }; + plugins: { disposed: string[]; failed: { provider: string; error: NormalizedError }[] }; +}; + +export type ProviderRuntimeDisposal = Readonly<{ + /** Disposes every provider runtime once; later calls return the first call's result. */ + dispose(): Promise; +}>; + +export function createProviderRuntimeDisposal( + runtimes: readonly Pick[], +): ProviderRuntimeDisposal { + let disposal: Promise | undefined; + return { + dispose: () => + (disposal ??= (async () => { + const report: ShutdownReport['plugins'] = { disposed: [], failed: [] }; + await Promise.all( + runtimes.map(async (runtime) => { + try { + await runtime.shutdown(); + report.disposed.push(runtime.provider); + } catch (error) { + report.failed.push({ provider: runtime.provider, error: normalizeError(error) }); + } + }), + ); + return report; + })()), + }; +} + +/** Undoes startup work in reverse order; a step that throws never stops the ones before it. */ +export type StartupRollback = Readonly<{ + onRollback(step: () => Promise | void): void; + run(): Promise; +}>; + +export function createStartupRollback(): StartupRollback { + const steps: Array<() => Promise | void> = []; + return { + onRollback: (step) => { + steps.push(step); + }, + run: async () => { + for (const step of steps.splice(0).reverse()) { + await Promise.resolve() + .then(step) + .catch(() => {}); + } + }, + }; +} + +export type WritableOutput = { + write: (chunk: string) => unknown; +}; + +async function settleDaemonTeardownStep(params: { + session: SessionState; + stderr: WritableOutput; + resource: 'app-log' | 'session' | 'lifecycle'; + teardown: () => Promise; +}): Promise { + const { session, stderr, resource, teardown } = params; + try { + await teardown(); + return true; + } catch (error) { + stderr.write( + `Daemon ${resource} teardown error (${session.name}): ${ + error instanceof Error ? error.message : String(error) + }\n`, + ); + return false; + } +} + +/** + * Daemon-shutdown teardown of one session: bounded resource cleanup (budget + * from {@link resolveDaemonSessionTeardownTimeoutMs}, resolved BEFORE cleanup + * starts since finalizing the recording detaches `session.screenRecording`), then the + * repair-commit finalization and session deletion. Cleanup failures — including + * a recorder that could not be finalized — surface on stderr instead of being + * silently swallowed. + */ +export async function teardownDaemonSessionForShutdown(params: { + ref: SessionRef; + sessionStore: SessionStore; + stderr: WritableOutput; + platformCleanup: PlatformResourceCleanup; + finalizeApplicationLifecycle?: ( + session: SessionState, + runtimeHints: RuntimeHintValues, + ) => Promise; + afterSuccessfulTeardown?: (session: SessionState) => Promise; +}): Promise { + const { + ref, + sessionStore, + stderr, + platformCleanup, + finalizeApplicationLifecycle, + afterSuccessfulTeardown, + } = params; + const current = sessionStore.resolveCurrent(ref); + const session = current ?? ref.session; + const runtimeHints = runtimeHintValues( + current ? sessionStore.getRuntimeHints(ref.address) : undefined, + ); + const timeoutMs = resolveDaemonSessionTeardownTimeoutMs(session); + // The ownership-fenced app-log side effect must settle while this process + // still owns the daemon lock. It is intentionally outside the generic + // teardown race so lock release and runtime shutdown cannot overtake it. + + const appLogTeardownSucceeded = await settleDaemonTeardownStep({ + session, + stderr, + resource: 'app-log', + teardown: async () => await stopSessionAppLog({ ref, sessionStore }), + }); + const sessionAfterAppLog = sessionStore.resolveCurrent(ref) ?? session; + const teardown = (async () => { + const genericTeardownSucceeded = await settleDaemonTeardownStep({ + session, + stderr, + resource: 'session', + teardown: async () => + await teardownSessionResources({ + appLog: 'already-settled', + ref, + sessionStore, + platformCleanup, + }), + }); + const lifecycleTeardownSucceeded = finalizeApplicationLifecycle + ? await settleDaemonTeardownStep({ + session, + stderr, + resource: 'lifecycle', + teardown: async () => + await finalizeApplicationLifecycle(sessionAfterAppLog, runtimeHints), + }) + : true; + return genericTeardownSucceeded && lifecycleTeardownSucceeded; + })(); + const genericTeardownSucceeded = await Promise.race([ + teardown, + sleep(timeoutMs).then(() => { + stderr.write(`Daemon session teardown timed out (${session.name}).\n`); + return false; + }), + ]); + const teardownSucceeded = appLogTeardownSucceeded && genericTeardownSucceeded; + // ADR 0012 decision 6, R7 + commit semantics (C2/C5a): commit the healed + // `.ad` iff the repair transaction completed, else leave a bounded + // `REPAIR_SESSION_EXPIRED` tombstone for the reaped-before-finalize case. + sessionStore.finalizeRepairTeardown(ref); + if (teardownSucceeded) await afterSuccessfulTeardown?.(session); + sessionStore.retire(ref); + return teardownSucceeded; +} + +export type RuntimeContext = Readonly<{ + run(fn: () => T): T; + /** Adds state the composition contributed and binds it for the rest of the start. */ + extend(handles: readonly RuntimeStateHandle[]): void; + dispose(): void; +}>; + +/** The per-runtime state of every module that keeps some, in the order it is entered. */ +function createDaemonRuntimeStates(): RuntimeStateHandle[] { + return [ + createRequestCancelRuntimeState(), + createProviderDeviceAdmissionRuntimeState(), + createInteractorResolutionRuntimeState(), + createArtifactTrackingRuntimeState(), + createResumableUploadRuntimeState(), + createMaterializedPathRuntimeState(), + ]; +} + +/** The context one daemon runtime enters: every module's per-runtime state, composed in order. */ +export function createRuntimeContext(): RuntimeContext { + const states = createDaemonRuntimeStates(); + return { + run: (fn) => states.reduceRight((next, state) => () => state.run(next), fn)(), + extend: (handles) => { + for (const handle of handles) { + states.push(handle); + handle.enter(); + } + }, + dispose: () => { + for (const state of states) state.dispose(); + }, + }; +} diff --git a/packages/daemon/src/server/http/transport.ts b/packages/daemon/src/server/http/transport.ts index c5e22a6329..83a5dbcb0a 100644 --- a/packages/daemon/src/server/http/transport.ts +++ b/packages/daemon/src/server/http/transport.ts @@ -1,5 +1,6 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { AppError, normalizeError, createRequestCanceledError } from '@agent-device/kernel/errors'; +import { AsyncResource } from 'node:async_hooks'; import net from 'node:net'; import type { Server as HttpServer } from 'node:http'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; @@ -27,8 +28,12 @@ export type DaemonServer = (net.Server | HttpServer) & { export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer { const sockets = new Set(); const server: DaemonServer = net.createServer((socket) => { + // An accepted socket loses the runtime that is listening, so its callbacks keep it explicitly. sockets.add(socket); - socket.on('close', () => sockets.delete(socket)); + socket.on( + 'close', + AsyncResource.bind(() => sockets.delete(socket)), + ); let buffer = ''; let inFlightRequests = 0; const activeRequestIds = new Set(); @@ -52,9 +57,9 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer }); }; socket.setEncoding('utf8'); - socket.on('close', cancelInFlightRequests); - socket.on('error', cancelInFlightRequests); - socket.on('data', async (chunk) => { + socket.on('close', AsyncResource.bind(cancelInFlightRequests)); + socket.on('error', AsyncResource.bind(cancelInFlightRequests)); + const onData = AsyncResource.bind(async (chunk: string) => { const parsed = consumeTextLines(buffer, chunk); buffer = parsed.buffer; for (const line of parsed.lines) { @@ -104,6 +109,7 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer } } }); + socket.on('data', onData); }); server.destroyConnections = () => { for (const socket of sockets) { diff --git a/packages/daemon/src/server/provider-lease-expiry.ts b/packages/daemon/src/server/provider-lease-expiry.ts index 038ef2b07d..349248ee1e 100644 --- a/packages/daemon/src/server/provider-lease-expiry.ts +++ b/packages/daemon/src/server/provider-lease-expiry.ts @@ -33,7 +33,15 @@ export type ExpiredProviderLeaseReleaser = { beginShutdown: () => void; release: (lease: DeviceLease) => Promise; retryPending: () => Promise; - drain: (timeoutMs: number) => Promise<{ pending: DeviceLease[]; released: DeviceLease[] }>; + /** + * `pending` holds every lease still unreleased; `unknown` is the part of it whose release was + * still in flight when the budget ran out, so its outcome is not known. + */ + drain: (timeoutMs: number) => Promise<{ + pending: DeviceLease[]; + unknown: DeviceLease[]; + released: DeviceLease[]; + }>; shutdown: () => void; }; @@ -206,6 +214,14 @@ export function createExpiredProviderLeaseReleaser(options: { await Promise.race([retryPending(), sleep(Math.max(0, timeoutMs))]); const result = { pending: [...pendingLiveLeases.values(), ...pendingRecoveryLeases.values()], + unknown: [ + ...[...pendingLiveLeases.values()].filter((lease) => + activeReleaseAttempts.has(`live:${lease.leaseId}`), + ), + ...[...pendingRecoveryLeases.values()].filter((lease) => + activeReleaseAttempts.has(`recovery:${lease.leaseId}`), + ), + ], released: [...shutdownReleasedLeases.values()], }; trackingShutdownReleases = false; diff --git a/packages/daemon/src/server/resumable-upload.ts b/packages/daemon/src/server/resumable-upload.ts index 74a86bd2ba..d7918ca88e 100644 --- a/packages/daemon/src/server/resumable-upload.ts +++ b/packages/daemon/src/server/resumable-upload.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; @@ -20,8 +22,42 @@ const RESUMABLE_UPLOAD_RESOURCE: TenantOwnedResourceKind = { label: 'Upload', expiredHint: `Resumable uploads expire ${RESUMABLE_UPLOAD_CLEANUP_TIMEOUT_MS / 60_000} minutes after the last received chunk. Start the upload again from the beginning.`, }; -const RESUMABLE_UPLOADS_BY_ID = new Map(); -const RESUMABLE_UPLOADS_BY_KEY = new Map(); +type ResumableUploadState = { + byId: Map; + byKey: Map; +}; + +const resumableUploadStorage = new AsyncLocalStorage(); +const createResumableUploadState = (): ResumableUploadState => ({ + byId: new Map(), + byKey: new Map(), +}); +const standaloneResumableUploads = createResumableUploadState(); +const resumableUploads = (): ResumableUploadState => + resumableUploadStorage.getStore() ?? standaloneResumableUploads; +const RESUMABLE_UPLOADS_BY_ID = () => resumableUploads().byId; +const RESUMABLE_UPLOADS_BY_KEY = () => resumableUploads().byKey; + +/** The resumable uploads one daemon runtime holds; disposing it stops their expiry timers. */ +export function createResumableUploadRuntimeState(): RuntimeStateHandle { + const owned = createResumableUploadState(); + return { + run: (fn) => resumableUploadStorage.run(owned, fn), + enter: () => resumableUploadStorage.enterWith(owned), + dispose: () => { + for (const entry of owned.byId.values()) { + clearTimeout(entry.timer); + try { + fs.rmSync(entry.tempDir, { recursive: true, force: true }); + } catch { + // best-effort cleanup only + } + } + owned.byId.clear(); + owned.byKey.clear(); + }, + }; +} export type BeginResumableUploadOptions = { baseUrl: string; @@ -62,8 +98,8 @@ export function beginResumableUpload(options: BeginResumableUploadOptions): { } { validateResumableUploadOptions(options); const key = buildResumableUploadKey(options); - const existingId = RESUMABLE_UPLOADS_BY_KEY.get(key); - const existing = existingId ? RESUMABLE_UPLOADS_BY_ID.get(existingId) : undefined; + const existingId = RESUMABLE_UPLOADS_BY_KEY().get(key); + const existing = existingId ? RESUMABLE_UPLOADS_BY_ID().get(existingId) : undefined; const entry = existing ?? createResumableUploadEntry(options, key); return { uploadId: entry.id, @@ -154,8 +190,8 @@ function createResumableUploadEntry( closed: false, finalized: false, }; - RESUMABLE_UPLOADS_BY_ID.set(id, entry); - RESUMABLE_UPLOADS_BY_KEY.set(key, id); + RESUMABLE_UPLOADS_BY_ID().set(id, entry); + RESUMABLE_UPLOADS_BY_KEY().set(key, id); refreshResumableUploadTimer(entry); return entry; } @@ -258,8 +294,8 @@ async function terminateEntry(entry: ResumableUploadEntry): Promise { function closeEntry(entry: ResumableUploadEntry): void { entry.closed = true; if (entry.timer) clearTimeout(entry.timer); - RESUMABLE_UPLOADS_BY_ID.delete(entry.id); - RESUMABLE_UPLOADS_BY_KEY.delete(entry.key); + RESUMABLE_UPLOADS_BY_ID().delete(entry.id); + RESUMABLE_UPLOADS_BY_KEY().delete(entry.key); } function requireResumableUpload( @@ -267,7 +303,7 @@ function requireResumableUpload( tenantId: string | undefined, ): ResumableUploadEntry { return requireTenantOwnedEntry( - RESUMABLE_UPLOADS_BY_ID, + RESUMABLE_UPLOADS_BY_ID(), uploadId, tenantId, RESUMABLE_UPLOAD_RESOURCE, diff --git a/packages/daemon/src/sessions/materialized-path-registry.ts b/packages/daemon/src/sessions/materialized-path-registry.ts index edc4d9e207..a350d054f1 100644 --- a/packages/daemon/src/sessions/materialized-path-registry.ts +++ b/packages/daemon/src/sessions/materialized-path-registry.ts @@ -1,4 +1,7 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; import crypto from 'node:crypto'; +import { rmSync } from 'node:fs'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -33,7 +36,30 @@ export type RetainedMaterializedPaths = { expiresAt: string; }; -const retainedPaths = new Map(); +const retainedPathsStorage = new AsyncLocalStorage>(); +const standaloneRetainedPaths = new Map(); +const retainedPaths = (): Map => + retainedPathsStorage.getStore() ?? standaloneRetainedPaths; + +/** The materialized paths one daemon runtime retains; disposing it stops their expiry timers. */ +export function createMaterializedPathRuntimeState(): RuntimeStateHandle { + const owned = new Map(); + return { + run: (fn) => retainedPathsStorage.run(owned, fn), + enter: () => retainedPathsStorage.enterWith(owned), + dispose: () => { + for (const entry of owned.values()) { + clearTimeout(entry.timer); + try { + rmSync(entry.rootPath, { recursive: true, force: true }); + } catch { + // best-effort cleanup only + } + } + owned.clear(); + }, + }; +} export async function retainMaterializedPaths(params: { installablePath: string; @@ -57,7 +83,7 @@ export async function retainMaterializedPaths(params: { const timer = setTimeout(() => { void expireRetainedMaterializedPaths(materializationId); }, ttlMs); - retainedPaths.set(materializationId, { + retainedPaths().set(materializationId, { rootPath, installablePath: retainedInstallablePath, archivePath: retainedArchivePath, @@ -83,7 +109,7 @@ export async function cleanupRetainedMaterializedPaths( tenantId?: string, ): Promise { const entry = requireTenantOwnedEntry( - retainedPaths, + retainedPaths(), materializationId, tenantId, MATERIALIZED_PATHS_RESOURCE, @@ -94,7 +120,7 @@ export async function cleanupRetainedMaterializedPaths( export async function cleanupRetainedMaterializedPathsForSession( sessionName: string, ): Promise { - const matchingEntries = Array.from(retainedPaths.entries()).filter( + const matchingEntries = Array.from(retainedPaths().entries()).filter( ([, entry]) => entry.sessionName === sessionName, ); await Promise.all( @@ -107,7 +133,7 @@ export async function cleanupRetainedMaterializedPathsForSession( // Daemon-internal expiry: skips the tenant check (the timer has no tenant // context) and must never reject — the caller is a bare setTimeout. async function expireRetainedMaterializedPaths(materializationId: string): Promise { - const entry = retainedPaths.get(materializationId); + const entry = retainedPaths().get(materializationId); if (!entry) return; try { await removeRetainedEntry(materializationId, entry); @@ -118,7 +144,7 @@ async function expireRetainedMaterializedPaths(materializationId: string): Promi async function removeRetainedEntry(materializationId: string, entry: RetainedEntry): Promise { clearTimeout(entry.timer); - retainedPaths.delete(materializationId); + retainedPaths().delete(materializationId); await fs.rm(entry.rootPath, { recursive: true, force: true }); } diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts index 03be15c5e5..f370fe7869 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts @@ -65,7 +65,17 @@ export type SessionOpenCommandInput = Readonly<{ reconcileOrphanedDeviceClaim: DeviceClaimReconciler; }>; -const firstSessionOpenLocks = new Map>(); +// One lock table per session store, which is to say per daemon runtime. +const firstSessionOpenLocksByStore = new WeakMap>>(); + +function firstSessionOpenLocks(sessionStore: SessionStore): Map> { + let locks = firstSessionOpenLocksByStore.get(sessionStore); + if (!locks) { + locks = new Map(); + firstSessionOpenLocksByStore.set(sessionStore, locks); + } + return locks; +} type OpenRuntimeAdmission = | Readonly<{ @@ -343,7 +353,7 @@ async function handleOpenCommand(params: SessionOpenCommandInput): Promise await openNewSessionWithDeviceClaim({ diff --git a/packages/host-kit/src/internal/request-cancel.ts b/packages/host-kit/src/internal/request-cancel.ts index 6d0b65516c..09a3122ea0 100644 --- a/packages/host-kit/src/internal/request-cancel.ts +++ b/packages/host-kit/src/internal/request-cancel.ts @@ -1,7 +1,48 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; -const canceledRequestIds = new Set(); -const requestAbortControllers = new Map(); +/** + * What one daemon runtime owns of a kind: the module keeps its state in an async-local slot the + * runtime enters, so two runtimes in one process never share it. Outside a runtime (a client + * process, a unit test) the module's own standalone state answers. + */ +export type RuntimeStateHandle = Readonly<{ + /** Runs `fn` as the runtime that owns this state. */ + run(fn: () => T): T; + /** Binds this state for the rest of the current async context, after the runtime is composed. */ + enter(): void; + /** Releases what the state holds (timers, handles). */ + dispose(): void; +}>; + +type RequestCancelState = { + canceledRequestIds: Set; + requestAbortControllers: Map; +}; + +const createRequestCancelState = (): RequestCancelState => ({ + canceledRequestIds: new Set(), + requestAbortControllers: new Map(), +}); +const requestCancelStorage = new AsyncLocalStorage(); +const standaloneRequestCancelState = createRequestCancelState(); +const requestCancelState = (): RequestCancelState => + requestCancelStorage.getStore() ?? standaloneRequestCancelState; +const canceledRequestIds = () => requestCancelState().canceledRequestIds; +const requestAbortControllers = () => requestCancelState().requestAbortControllers; + +export function createRequestCancelRuntimeState(): RuntimeStateHandle { + const owned = createRequestCancelState(); + return { + run: (fn) => requestCancelStorage.run(owned, fn), + enter: () => requestCancelStorage.enterWith(owned), + dispose: () => { + for (const controller of owned.requestAbortControllers.values()) controller.abort(); + owned.requestAbortControllers.clear(); + owned.canceledRequestIds.clear(); + }, + }; +} export type RequestAbortRegistration = { requestId: string; @@ -55,16 +96,16 @@ export function registerRequestAbort( requestId: string | undefined, ): RequestAbortRegistration | undefined { if (!requestId) return undefined; - if (requestAbortControllers.has(requestId)) { + if (requestAbortControllers().has(requestId)) { throw new AppError('INVALID_ARGS', `Request ID is already in flight: ${requestId}`, { requestId, reason: 'duplicate_request_id', }); } - evictOldestEntries(requestAbortControllers); + evictOldestEntries(requestAbortControllers()); const controller = new AbortController(); - requestAbortControllers.set(requestId, controller); - if (canceledRequestIds.has(requestId)) { + requestAbortControllers().set(requestId, controller); + if (canceledRequestIds().has(requestId)) { controller.abort(); } return { requestId, controller }; @@ -72,13 +113,13 @@ export function registerRequestAbort( export function markRequestCanceled(requestId: string | undefined): void { if (!requestId) return; - evictOldestSetEntries(canceledRequestIds); - canceledRequestIds.add(requestId); + evictOldestSetEntries(canceledRequestIds()); + canceledRequestIds().add(requestId); // Abort WITH the typed error as the reason: every `signal.throwIfAborted()`, // fetch, and `throw signal.reason` downstream then surfaces the canceled // request as such, instead of a bare DOMException that normalizes to UNKNOWN. // No call site has to remember the factory — the signal carries it. - requestAbortControllers.get(requestId)?.abort(createRequestCanceledError()); + requestAbortControllers().get(requestId)?.abort(createRequestCanceledError()); } export function clearRequestCanceled( @@ -89,12 +130,12 @@ export function clearRequestCanceled( if ( registration && (registration.requestId !== requestId || - requestAbortControllers.get(requestId) !== registration.controller) + requestAbortControllers().get(requestId) !== registration.controller) ) { return; } - canceledRequestIds.delete(requestId); - requestAbortControllers.delete(requestId); + canceledRequestIds().delete(requestId); + requestAbortControllers().delete(requestId); } export function clearRequestAbortRegistration( @@ -106,12 +147,12 @@ export function clearRequestAbortRegistration( export function isRequestCanceled(requestId: string | undefined): boolean { if (!requestId) return false; - return canceledRequestIds.has(requestId); + return canceledRequestIds().has(requestId); } export function getRequestSignal(requestId: string | undefined): AbortSignal | undefined { if (!requestId) return undefined; - return requestAbortControllers.get(requestId)?.signal; + return requestAbortControllers().get(requestId)?.signal; } export function throwIfRequestCanceled(requestId: string | undefined): void { diff --git a/packages/host-kit/src/request.ts b/packages/host-kit/src/request.ts index ee98a01e7b..5ae44f36fc 100644 --- a/packages/host-kit/src/request.ts +++ b/packages/host-kit/src/request.ts @@ -1,5 +1,7 @@ export { clearRequestAbortRegistration, + createRequestCancelRuntimeState, + type RuntimeStateHandle, clearRequestCanceled, getRequestSignal, isRequestCanceled, diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index 6790d89a7c..f84ea4b4a1 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -42,6 +42,7 @@ import { visitXmlPlistEntries } from './plist-xml.ts'; import { getRunnerDeviceClaimAuthorityProbe, getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, } from './runner-owner-state.ts'; import { buildSimctlArgsForDevice, simulatorAddressFor } from './simctl.ts'; import { @@ -108,5 +109,6 @@ export const appleRunnerHost: AppleRunnerHost = { resolveIosPhysicalDeviceControl, visitXmlPlistEntries, leaseOwnerStateDir: getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, hasDeviceClaimAuthority: (device) => getRunnerDeviceClaimAuthorityProbe()?.(device) ?? false, }; diff --git a/packages/platform-apple/src/core/runner-owner-state.test.ts b/packages/platform-apple/src/core/runner-owner-state.test.ts new file mode 100644 index 0000000000..fa4de52b7a --- /dev/null +++ b/packages/platform-apple/src/core/runner-owner-state.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from 'vitest'; +import { + createAppleRunnerRuntimeState, + getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, + setRunnerLeaseOwnerStateDir, +} from './runner-owner-state.ts'; + +const SLOT = Symbol('test-slot'); + +test('each runtime keeps its own owner state and registries', () => { + const first = createAppleRunnerRuntimeState(); + const second = createAppleRunnerRuntimeState(); + + first.run(() => { + setRunnerLeaseOwnerStateDir(' /state/first '); + runnerRuntimeSlot(SLOT, () => new Map()).set('device', 'first'); + }); + second.run(() => { + setRunnerLeaseOwnerStateDir('/state/second'); + }); + + expect(first.run(() => getRunnerLeaseOwnerStateDir())).toBe('/state/first'); + expect(second.run(() => getRunnerLeaseOwnerStateDir())).toBe('/state/second'); + expect( + first.run(() => runnerRuntimeSlot(SLOT, () => new Map()).get('device')), + ).toBe('first'); + expect(second.run(() => runnerRuntimeSlot(SLOT, () => new Map()).size)).toBe(0); + // Nothing leaked into the standalone state the code outside any runtime reads. + expect(getRunnerLeaseOwnerStateDir()).toBeUndefined(); +}); + +test('disposing a runtime drops its registries', () => { + const runtime = createAppleRunnerRuntimeState(); + runtime.run(() => runnerRuntimeSlot(SLOT, () => new Map([['device', 'owned']]))); + + runtime.dispose(); + + expect(runtime.run(() => runnerRuntimeSlot(SLOT, () => new Map()).size)).toBe(0); +}); diff --git a/packages/platform-apple/src/core/runner-owner-state.ts b/packages/platform-apple/src/core/runner-owner-state.ts index e44dddd94e..534f58e3ca 100644 --- a/packages/platform-apple/src/core/runner-owner-state.ts +++ b/packages/platform-apple/src/core/runner-owner-state.ts @@ -1,20 +1,6 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; import type { DeviceInfo } from '@agent-device/kernel/device'; - -/** - * Daemon-owned lease-owner state directory for the Apple runner. Kept in a - * dedicated tiny module so daemon startup can set it without loading the - * runner client's implementation closure; the runner package reads it through - * its host port at lease time. - */ -let runnerLeaseOwnerStateDir: string | undefined; - -export function setRunnerLeaseOwnerStateDir(stateDir: string | undefined): void { - runnerLeaseOwnerStateDir = stateDir?.trim() || undefined; -} - -export function getRunnerLeaseOwnerStateDir(): string | undefined { - return runnerLeaseOwnerStateDir; -} +import type { RuntimeStateHandle } from '@agent-device/host-kit/request'; /** * Daemon-owned device-claim arbitration probe: does the embedding process hold @@ -26,14 +12,61 @@ export function getRunnerLeaseOwnerStateDir(): string | undefined { */ export type RunnerDeviceClaimAuthorityProbe = (device: DeviceInfo) => boolean; -let runnerDeviceClaimAuthorityProbe: RunnerDeviceClaimAuthorityProbe | undefined; +/** + * What one daemon runtime owns of the Apple runner: the lease-owner state directory and claim probe + * its daemon set, and the registries (sessions, locks, recycle ledger) the runner modules keep in + * {@link runnerRuntimeSlot}s. Kept in a dedicated tiny module so daemon startup can reach it + * without loading the runner client's implementation closure; the runner package reads it through + * its host port. + */ +type RunnerRuntimeState = { + leaseOwnerStateDir: string | undefined; + deviceClaimAuthorityProbe: RunnerDeviceClaimAuthorityProbe | undefined; + slots: Map; +}; + +const createRunnerRuntimeState = (): RunnerRuntimeState => ({ + leaseOwnerStateDir: undefined, + deviceClaimAuthorityProbe: undefined, + slots: new Map(), +}); + +const runnerRuntimeStorage = new AsyncLocalStorage(); +// Outside a daemon runtime (a package test, a tool that drives the runner directly) this answers. +const standaloneRunnerRuntime = createRunnerRuntimeState(); +const runnerRuntimeState = (): RunnerRuntimeState => + runnerRuntimeStorage.getStore() ?? standaloneRunnerRuntime; + +export function createAppleRunnerRuntimeState(): RuntimeStateHandle { + const owned = createRunnerRuntimeState(); + return { + run: (fn) => runnerRuntimeStorage.run(owned, fn), + enter: () => runnerRuntimeStorage.enterWith(owned), + dispose: () => owned.slots.clear(), + }; +} + +/** One piece of runner state this daemon runtime owns, created on first use. */ +export function runnerRuntimeSlot(key: symbol, create: () => T): T { + const { slots } = runnerRuntimeState(); + if (!slots.has(key)) slots.set(key, create()); + return slots.get(key) as T; +} + +export function setRunnerLeaseOwnerStateDir(stateDir: string | undefined): void { + runnerRuntimeState().leaseOwnerStateDir = stateDir?.trim() || undefined; +} + +export function getRunnerLeaseOwnerStateDir(): string | undefined { + return runnerRuntimeState().leaseOwnerStateDir; +} export function setRunnerDeviceClaimAuthorityProbe( probe: RunnerDeviceClaimAuthorityProbe | undefined, ): void { - runnerDeviceClaimAuthorityProbe = probe; + runnerRuntimeState().deviceClaimAuthorityProbe = probe; } export function getRunnerDeviceClaimAuthorityProbe(): RunnerDeviceClaimAuthorityProbe | undefined { - return runnerDeviceClaimAuthorityProbe; + return runnerRuntimeState().deviceClaimAuthorityProbe; } diff --git a/packages/platform-apple/src/runner-owner-facade.ts b/packages/platform-apple/src/runner-owner-facade.ts index 9b683958d8..123e115748 100644 --- a/packages/platform-apple/src/runner-owner-facade.ts +++ b/packages/platform-apple/src/runner-owner-facade.ts @@ -1,4 +1,5 @@ export { + createAppleRunnerRuntimeState, setRunnerDeviceClaimAuthorityProbe, setRunnerLeaseOwnerStateDir, type RunnerDeviceClaimAuthorityProbe, diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts index a5ecc5270f..e8436b5f97 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts @@ -381,7 +381,7 @@ test('graceful shutdown stops idle retained generations and hands off active rea const active = await ensureReadyRunner(activeDevice); await closeFinalizationLifecycle(idleDevice, []).finalizeApplicationClose(closeInput(true)); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(active.state, 'stopped'); assert.equal(idle.state, 'ready'); await stopAllIosRunnerSessions(); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts index ab7a494392..2251d23ec9 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts @@ -343,7 +343,7 @@ test('shutdown detach moves a handed-off session to stopped without killing its assert.ok(runnerPid); runnerStateTransitions.length = 0; - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(session.state, 'stopped'); assert.deepEqual(runnerStateTransitions, ['stopped']); @@ -361,7 +361,7 @@ test('a scoped simulator-set session hands off like one in the default set', asy const session = await ensureRunnerSession(device, {}); await serveOneCommand(device, session); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(session.state, 'stopped'); assert.match(leaseRaw(device.id), /"ownerToken": "detached-owner-/); @@ -393,7 +393,7 @@ test('a runner that served a command is handed off on the physical lane', async runnerStateTransitions.length = 0; const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); }); assert.equal(session.state, 'stopped'); @@ -418,7 +418,7 @@ test('a runner that never served a command is torn down instead of handed off', assert.equal(session.state, 'starting'); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.deepEqual(readRunnerSessionLiveness(device.id), { @@ -449,7 +449,7 @@ test('a runner reporting main-thread work still draining is not handed off', asy assert.equal(session.state, 'ready'); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"main_thread_occupied"/); @@ -484,7 +484,7 @@ test('a session that still owes a response is not handed off', async () => { assert.equal(session.commandCharges.outstandingChargeCount, 1); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"command_in_flight"/); @@ -521,7 +521,7 @@ test('a command abandoned by a cancelled transport keeps the runner occupied', a assert.equal(session.commandCharges.hasAbandonedCharges, true); const refused = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(refused, /"reason":"command_in_flight"/); // The refusal this issue is about: nothing is awaited any more, and only terminal evidence for that @@ -535,7 +535,7 @@ test('a command abandoned by a cancelled transport keeps the runner occupied', a // `runner-client-recovery-wiring.test.ts`, where the probe reaches a real session. await serveOneCommand(device, session); assert.equal(session.commandCharges.hasOutstandingCharges, false); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); }); test('the macOS host runner is never handed off, although it is kind device', async () => { @@ -544,7 +544,7 @@ test('the macOS host runner is never handed off, although it is kind device', as await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"macos_host"/); @@ -563,7 +563,7 @@ test('a physical tvOS runner is never handed off, although it is kind device too await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"physical_non_ios_os"/); @@ -580,7 +580,7 @@ test('a usbmux-only xctest backend runner is never handed off', async () => { await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"xctest_backend"/); @@ -597,7 +597,7 @@ test("the handoff releases this daemon's log bookkeeping only after the lease sa tokenWhenReleased.push(leaseRaw(device.id)); }; - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(tokenWhenReleased.length, 1); assert.match(tokenWhenReleased[0]!, /"ownerToken": "detached-owner-/); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts new file mode 100644 index 0000000000..6d0e3db2c7 --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { beforeEach, test, vi } from 'vitest'; +import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; +import { createAppleRunnerRuntimeState } from '../../core/runner-owner-state.ts'; +import { appleRunnerTestHost } from '../test-host.ts'; +import { registerRunnerPrepProcess } from '../runner-artifact.ts'; +import { stopAllIosRunnerSessions } from '../runner-session.ts'; +import { openRunnerStartAdmission, runnerPrepProcessChildren } from '../runner-xctestrun.ts'; + +const mockSignalProcessGroupBestEffort = vi.fn(); +const mockRunAppleToolCommand = vi.fn(); + +beforeEach(() => { + vi.resetAllMocks(); + mockRunAppleToolCommand.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); + appleRunnerTestHost.update({ + runAppleToolCommand: mockRunAppleToolCommand, + signalProcessGroupBestEffort: mockSignalProcessGroupBestEffort, + signalPidsBestEffort: vi.fn(), + }); +}); + +function prepChild(pid: number): ExecBackgroundResult['child'] { + return Object.assign(new EventEmitter(), { + pid, + exitCode: null, + }) as ExecBackgroundResult['child']; +} + +test("stopping every runner of one runtime leaves another runtime's prep build and admission alone", async () => { + const runtimeA = createAppleRunnerRuntimeState(); + const runtimeB = createAppleRunnerRuntimeState(); + const deviceId = 'runner-runtime-isolation-sim'; + const buildA = prepChild(6101); + const buildB = prepChild(6102); + + runtimeA.run(() => { + registerRunnerPrepProcess(deviceId, buildA, openRunnerStartAdmission(deviceId)); + }); + const admissionB = runtimeB.run(() => { + const admission = openRunnerStartAdmission(deviceId); + registerRunnerPrepProcess(deviceId, buildB, admission); + return admission; + }); + + await runtimeA.run(async () => await stopAllIosRunnerSessions()); + + const signaled = mockSignalProcessGroupBestEffort.mock.calls.map(([pid]) => pid as number); + assert.ok(signaled.includes(6101), 'runtime A stopped its own build'); + assert.ok(!signaled.includes(6102), "runtime A never signaled runtime B's build"); + assert.equal( + runtimeA.run(() => runnerPrepProcessChildren(deviceId).length), + 0, + ); + assert.equal( + runtimeB.run(() => runnerPrepProcessChildren(deviceId).length), + 1, + ); + assert.equal(admissionB.admitted, true, "runtime B's start admission stays open"); +}); diff --git a/packages/platform-apple/src/runner/host.ts b/packages/platform-apple/src/runner/host.ts index 37be783617..dbc631f8bc 100644 --- a/packages/platform-apple/src/runner/host.ts +++ b/packages/platform-apple/src/runner/host.ts @@ -97,6 +97,8 @@ export type AppleRunnerHost = Pick< resolveIosPhysicalDeviceControl: (device: DeviceInfo) => IosPhysicalDeviceRunnerControl; /** Daemon-owned lease owner state directory. */ leaseOwnerStateDir: typeof AppleRunnerOwnerState.getRunnerLeaseOwnerStateDir; + /** One registry the daemon runtime owns (sessions, locks, ledgers), created on first use. */ + runnerRuntimeSlot: typeof AppleRunnerOwnerState.runnerRuntimeSlot; /** * Daemon-owned device-claim arbitration probe: true only while the embedding process holds * the host-global local device claim for exactly this device (matched by canonical @@ -173,6 +175,7 @@ export const classifyOwnerLiveness = delegate('classifyOwnerLiveness'); export const readHostCpuArchSync = delegate('readHostCpuArchSync'); export const findProjectRoot = delegate('findProjectRoot'); export const readVersion = delegate('readVersion'); +export const runnerRuntimeSlot = delegate('runnerRuntimeSlot'); export const acquireProcessLock = delegate('acquireProcessLock'); export const withProcessLock = delegate('withProcessLock'); export const publishFileSync = delegate('publishFileSync'); diff --git a/packages/platform-apple/src/runner/runner-adoption.ts b/packages/platform-apple/src/runner/runner-adoption.ts index 3048c2a6a7..e90af5285b 100644 --- a/packages/platform-apple/src/runner/runner-adoption.ts +++ b/packages/platform-apple/src/runner/runner-adoption.ts @@ -83,9 +83,9 @@ export function isIosRunnerDetachEnabled(env: NodeJS.ProcessEnv = process.env): export async function detachRunnerSessionsForShutdown( sessions: ReadonlyMap, onDetached: (deviceId: string, session: RunnerSession) => void, -): Promise { - if (!isIosRunnerDetachEnabled()) return 0; - let detached = 0; +): Promise { + if (!isIosRunnerDetachEnabled()) return []; + const detached: string[] = []; for (const [deviceId, session] of sessions) { const outcome = detachRunnerSessionForShutdown(session); if (!outcome.detached) { @@ -106,7 +106,7 @@ export async function detachRunnerSessionsForShutdown( }); continue; } - detached += 1; + detached.push(deviceId); onDetached(deviceId, session); emitDiagnostic({ level: 'info', diff --git a/packages/platform-apple/src/runner/runner-artifact.ts b/packages/platform-apple/src/runner/runner-artifact.ts index 2db0996401..dbf3b4a6df 100644 --- a/packages/platform-apple/src/runner/runner-artifact.ts +++ b/packages/platform-apple/src/runner/runner-artifact.ts @@ -15,6 +15,7 @@ import { emitRequestProgress, findProjectRoot, isCommandTimeoutError, + runnerRuntimeSlot, } from './host.ts'; import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -119,7 +120,10 @@ type DeviceStartState = { inFlight: Set; }; -const deviceStartStates = new Map(); +const DEVICE_START_STATES_SLOT = Symbol('apple-runner-device-start-states'); +// Per daemon runtime: a runtime's teardown retires only the start admissions it opened. +const deviceStartStates = () => + runnerRuntimeSlot(DEVICE_START_STATES_SLOT, () => new Map()); /** * Opens the admission one start answers to, taken when a caller asks for a runner. Registration is @@ -144,7 +148,7 @@ export function openRunnerStartLoopAdmission(deviceId: string): RunnerStartAdmis /** The start settled: its token leaves the device's in-flight set. Verdicts live on the token. */ export function finishRunnerStartAdmission(admission: RunnerStartAdmission): void { (admission as StartAdmission).retryPending = false; - const state = deviceStartStates.get(admission.deviceId); + const state = deviceStartStates().get(admission.deviceId); if (!state) return; state.inFlight.delete(admission as StartAdmission); forgetDeviceStartState(admission.deviceId, state); @@ -180,7 +184,10 @@ export function fenceRunnerStartAdmissionsForTeardown(deviceId: string): () => v * build past the sweep that is about to stop it. */ export function retireAllRunnerStartAdmissions(additionalDeviceIds?: Iterable): () => void { - const deviceIds = new Set([...deviceStartStates.keys(), ...(additionalDeviceIds ?? [])]); + const deviceIds = new Set([ + ...deviceStartStates().keys(), + ...(additionalDeviceIds ?? []), + ]); const settles = [...deviceIds].map((deviceId) => fenceRunnerStartAdmissionsForTeardown(deviceId)); return () => { for (const settle of settles.splice(0)) settle(); @@ -193,7 +200,7 @@ export function retireAllRunnerStartAdmissions(additionalDeviceIds?: Iterable 0; + return (deviceStartStates().get(deviceId)?.pendingTeardowns ?? 0) > 0; } /** @@ -299,17 +306,17 @@ export function releaseRunnerStartWaiter( } function stateForDevice(deviceId: string): DeviceStartState { - const existing = deviceStartStates.get(deviceId); + const existing = deviceStartStates().get(deviceId); if (existing) return existing; const state: DeviceStartState = { pendingTeardowns: 0, inFlight: new Set() }; - deviceStartStates.set(deviceId, state); + deviceStartStates().set(deviceId, state); return state; } /** Nothing fences the device and no start is in flight on it once the last of both is gone. */ function forgetDeviceStartState(deviceId: string, state: DeviceStartState): void { if (state.pendingTeardowns <= 0 && state.inFlight.size === 0) { - deviceStartStates.delete(deviceId); + deviceStartStates().delete(deviceId); } } @@ -360,7 +367,10 @@ type RunnerPrepProcess = Readonly<{ child: ExecBackgroundResult['child']; }>; -const runnerPrepProcessLedger = new Set(); +const RUNNER_PREP_LEDGER_SLOT = Symbol('apple-runner-prep-process-ledger'); +// Per daemon runtime: a runtime's stop signals only the prep children it started. +const runnerPrepProcessLedger = () => + runnerRuntimeSlot(RUNNER_PREP_LEDGER_SLOT, () => new Set()); /** * Records a prep subprocess (`xcodebuild build-for-testing`) against the device it builds for and @@ -377,9 +387,9 @@ export function registerRunnerPrepProcess( startAdmission?: RunnerStartAdmission, ): void { const entry: RunnerPrepProcess = { deviceId, startAdmission, child }; - runnerPrepProcessLedger.add(entry); + runnerPrepProcessLedger().add(entry); child.on('close', () => { - runnerPrepProcessLedger.delete(entry); + runnerPrepProcessLedger().delete(entry); }); } @@ -426,14 +436,14 @@ export function markRunnerStartRetryPending(admission: RunnerStartAdmission): vo } function prepProcessEntries(deviceId?: string): readonly RunnerPrepProcess[] { - return [...runnerPrepProcessLedger].filter( + return [...runnerPrepProcessLedger()].filter( (entry) => deviceId === undefined || entry.deviceId === deviceId, ); } export function forgetRunnerPrepProcess(child: ExecBackgroundResult['child']): void { - for (const entry of runnerPrepProcessLedger) { - if (entry.child === child) runnerPrepProcessLedger.delete(entry); + for (const entry of runnerPrepProcessLedger()) { + if (entry.child === child) runnerPrepProcessLedger().delete(entry); } } diff --git a/packages/platform-apple/src/runner/runner-cache.ts b/packages/platform-apple/src/runner/runner-cache.ts index fbb60aba68..d1d8566886 100644 --- a/packages/platform-apple/src/runner/runner-cache.ts +++ b/packages/platform-apple/src/runner/runner-cache.ts @@ -8,6 +8,7 @@ import { withProcessLock, isEnvTruthy, findProjectRoot, + runnerRuntimeSlot, } from './host.ts'; import { RUNNER_CACHE_METADATA_FILE, @@ -38,7 +39,9 @@ export { requireRunnerPhaseRemainingMs, type RunnerPhaseBudget } from './runner- const RUNNER_XCTESTRUN_CACHE_LOCK_TIMEOUT_MS = 10 * 60_000; const RUNNER_XCTESTRUN_CACHE_LOCK_POLL_MS = 100; -const badRunnerArtifactsForRun = new Set(); +const BAD_ARTIFACTS_SLOT = Symbol('apple-runner-bad-artifacts'); +const badRunnerArtifactsForRun = () => + runnerRuntimeSlot(BAD_ARTIFACTS_SLOT, () => new Set()); export type RunnerXctestrunCacheKind = 'exact' | 'miss' | 'external'; @@ -102,7 +105,7 @@ export async function markRunnerXctestrunArtifactBadForRun( return; } - badRunnerArtifactsForRun.add(artifact.derived); + badRunnerArtifactsForRun().add(artifact.derived); await withProcessLock({ acquire: () => acquireRunnerXctestrunCacheLock(artifact.derived), task: async () => { @@ -139,7 +142,7 @@ function resolveRunnerXctestrunCacheLockPath(derived: string): string { } export function cleanRunnerDerivedBeforeEvaluation(derived: string, forceRebuild: boolean): void { - if (!shouldCleanDerived() && !forceRebuild && !badRunnerArtifactsForRun.has(derived)) { + if (!shouldCleanDerived() && !forceRebuild && !badRunnerArtifactsForRun().has(derived)) { return; } emitRunnerXctestrunDecision('clean', forceRebuild ? 'forced_rebuild' : 'forced_clean', { @@ -147,7 +150,7 @@ export function cleanRunnerDerivedBeforeEvaluation(derived: string, forceRebuild }); assertSafeDerivedCleanup(derived); cleanRunnerDerivedArtifacts(derived); - badRunnerArtifactsForRun.delete(derived); + badRunnerArtifactsForRun().delete(derived); } /** diff --git a/packages/platform-apple/src/runner/runner-recycle-ledger.ts b/packages/platform-apple/src/runner/runner-recycle-ledger.ts index 149e681ea3..8af8fbd867 100644 --- a/packages/platform-apple/src/runner/runner-recycle-ledger.ts +++ b/packages/platform-apple/src/runner/runner-recycle-ledger.ts @@ -1,5 +1,5 @@ import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; -import { emitDiagnostic } from './host.ts'; +import { emitDiagnostic, runnerRuntimeSlot } from './host.ts'; import type { RunnerCommand } from './runner-contract.ts'; // Caps how many times one daemon request may pay for a full runner recycle @@ -19,7 +19,9 @@ type RunnerRecycleLedgerEntry = { lastAtMs: number; }; -const ledger = new Map(); +const LEDGER_SLOT = Symbol('apple-runner-recycle-ledger'); +const ledgerEntries = (): Map => + runnerRuntimeSlot(LEDGER_SLOT, () => new Map()); /** Requests are the recycle scope; direct calls without a request fall back to the command id. */ export function runnerRecycleLedgerKey( @@ -43,7 +45,7 @@ export function markRunnerRequestTouchedSession(key: string | undefined): void { export function hasRunnerRequestTouchedSession(key: string | undefined): boolean { if (!key) return false; - return ledger.get(key)?.touchedSession === true; + return ledgerEntries().get(key)?.touchedSession === true; } /** @@ -108,28 +110,28 @@ export function buildRunnerRecycleBudgetExhaustedError( * @internal Test isolation hook for process-global runner recycle accounting. */ export function resetRunnerRecycleLedgerForTests(): void { - ledger.clear(); + ledgerEntries().clear(); } function readEntry(key: string): RunnerRecycleLedgerEntry { pruneLedger(); - return ledger.get(key) ?? { touchedSession: false, recycles: 0, lastAtMs: Date.now() }; + return ledgerEntries().get(key) ?? { touchedSession: false, recycles: 0, lastAtMs: Date.now() }; } function writeEntry(key: string, entry: RunnerRecycleLedgerEntry): void { entry.lastAtMs = Date.now(); - ledger.delete(key); - ledger.set(key, entry); + ledgerEntries().delete(key); + ledgerEntries().set(key, entry); } function pruneLedger(): void { const cutoff = Date.now() - LEDGER_ENTRY_TTL_MS; - for (const [key, entry] of ledger) { - if (entry.lastAtMs < cutoff) ledger.delete(key); + for (const [key, entry] of ledgerEntries()) { + if (entry.lastAtMs < cutoff) ledgerEntries().delete(key); } - while (ledger.size > LEDGER_MAX_ENTRIES) { - const oldest = ledger.keys().next().value; + while (ledgerEntries().size > LEDGER_MAX_ENTRIES) { + const oldest = ledgerEntries().keys().next().value; if (oldest === undefined) break; - ledger.delete(oldest); + ledgerEntries().delete(oldest); } } diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 16b00c4043..243436a06f 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -1,5 +1,6 @@ import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { + runnerRuntimeSlot, withKeyedLock, Deadline, emitRequestProgress, @@ -84,12 +85,17 @@ export type { RunnerSession } from './runner-session-types.ts'; export type RunnerSessionOptions = AppleRunnerLifecycleOptions; -const runnerSessions = new Map(); -const runnerSessionLocks = new Map>(); +const RUNNER_SESSIONS_SLOT = Symbol('apple-runner-sessions'); +const RUNNER_SESSION_LOCKS_SLOT = Symbol('apple-runner-session-locks'); +// The daemon runtime's registries: two runtimes in one process never see each other's runners. +const runnerSessions = () => + runnerRuntimeSlot(RUNNER_SESSIONS_SLOT, () => new Map()); +const runnerSessionLocks = () => + runnerRuntimeSlot(RUNNER_SESSION_LOCKS_SLOT, () => new Map>()); const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; function withRunnerSessionLock(deviceId: string, task: () => Promise): Promise { - return withKeyedLock(runnerSessionLocks, deviceId, task); + return withKeyedLock(runnerSessionLocks(), deviceId, task); } export async function ensureRunnerSession( @@ -122,7 +128,7 @@ export async function ensureRunnerSession( // a joiner's cancellation can never outvote the live owner and stop its build (#3220). const releaseOwnerInterest = reserveRunnerStartOwnerInterest(startAdmission, options); try { - const existing = runnerSessions.get(device.id); + const existing = runnerSessions().get(device.id); if (existing) { assertExpectedRunnerSession(existing, options.expectedRunnerSessionId); const reusable = await resolveReusableRunnerSession(device, existing, budget.phase); @@ -195,7 +201,7 @@ async function startRunnerSessionWithLease( // gate: a start fenced by a teardown registers its runner into no device (#3220). The // runner itself keeps its lease for the next open to adopt. assertRunnerStartAdmitsPreparation(device.id, startAdmission); - runnerSessions.set(device.id, adopted); + runnerSessions().set(device.id, adopted); return adopted; } assertRunnerSessionMayStart(options.expectedRunnerSessionId); @@ -369,7 +375,7 @@ async function startRunnerSessionWithLease( }); throw error; } - runnerSessions.set(device.id, session); + runnerSessions().set(device.id, session); return session; } @@ -560,7 +566,7 @@ async function uninstallStaleSimulatorRunnerBundle( * registered, which is its own answer: there is no session to wait for or tear down. */ export function readRunnerSessionLiveness(deviceId: string): RunnerSessionRegistration | null { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session) return null; return { sessionId: session.sessionId, @@ -580,7 +586,7 @@ export async function invalidateRunnerSession( reason: string, ): Promise { await withRunnerSessionLock(session.deviceId, async () => { - if (runnerSessions.get(session.deviceId) !== session) return; + if (runnerSessions().get(session.deviceId) !== session) return; // A session already being torn down, or already torn down, is never disposed a second time // for a later reason; the reason-coded diagnostic below reports why this call was made. if (!canWorkWithRunnerSession(session)) return; @@ -605,15 +611,15 @@ async function stopRunnerSessionInternal( sessionOverride?: RunnerSession, options: RunnerDisposalOptions = {}, ): Promise { - const session = sessionOverride ?? runnerSessions.get(deviceId); + const session = sessionOverride ?? runnerSessions().get(deviceId); if (!session) return; // Once disposal has begun or finished, this session has no runner to wait on; a repeat stop // would only re-signal a process that is already leaving and re-emit a teardown for a reason // that has nothing left to tear down. if (!canWorkWithRunnerSession(session)) return; await disposeRunnerSession(session, options); - if (runnerSessions.get(deviceId) === session) { - runnerSessions.delete(deviceId); + if (runnerSessions().get(deviceId) === session) { + runnerSessions().delete(deviceId); } } @@ -623,7 +629,8 @@ async function stopRetainedRunner( reason: 'idle_timeout' | 'listener_lost', ): Promise { await withRunnerSessionLock(session.deviceId, async () => { - if (runnerSessions.get(session.deviceId) !== session || session.retention !== retention) return; + if (runnerSessions().get(session.deviceId) !== session || session.retention !== retention) + return; emitDiagnostic({ level: reason === 'listener_lost' ? 'warn' : 'info', phase: reason === 'listener_lost' ? 'ios_runner_warm_stop' : 'ios_runner_idle_stop', @@ -652,7 +659,7 @@ export async function releaseSpeculativeIosRunnerSession(deviceId: string): Prom // Under the session lock: a prewarm still starting holds it and registers its session only // when the start completes, so the release queues behind that start instead of missing it. return await withRunnerSessionLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session?.speculative) return false; emitDiagnostic({ level: 'debug', @@ -684,7 +691,7 @@ async function stopIosRunnerSessionDevice( try { await withRunnerSessionLock(deviceId, async () => { await withRunnerLeaseLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); await stopRunnerSessionInternal(deviceId, session, { leaseLockHeld: true, graceful: @@ -706,13 +713,13 @@ export async function releaseIosRunnerOnClose( deviceId: string, options: { retain: boolean }, ): Promise { - const candidate = runnerSessions.get(deviceId); + const candidate = runnerSessions().get(deviceId); const busyAtClose = isRunnerMainThreadOccupied(candidate); const settleFence = fenceRunnerStartAdmissionsForTeardown(deviceId); try { if (options.retain && candidate?.state === 'ready' && !busyAtClose) { const retained = await withRunnerSessionLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session) return false; const { retainRunnerSession } = await import('./runner-retention.ts'); if ( @@ -751,15 +758,15 @@ export async function abortAllIosRunnerSessions(): Promise { // The same fence a close raises, for every device at once: the prep sweep the abort performs is // one-shot, so an in-flight start that would answer it with another build is refused before the // sweep runs, and stays refused until the sweep settles (#3220). - const activeSessions = Array.from(runnerSessions.values()); + const activeSessions = Array.from(runnerSessions().values()); const settleFences = retireAllRunnerStartAdmissions( activeSessions.map((session) => session.deviceId), ); try { await abortRunnerSessionsAndPrepProcesses(activeSessions); for (const session of activeSessions) { - if (runnerSessions.get(session.deviceId) === session) { - runnerSessions.delete(session.deviceId); + if (runnerSessions().get(session.deviceId) === session) { + runnerSessions().delete(session.deviceId); } } } finally { @@ -771,26 +778,29 @@ export async function abortAllIosRunnerSessions(): Promise { // The detach decision itself lives in runner-adoption.ts beside the adoption it hands off to // (#2681); detached active generations leave the registry. Idle retained generations stay // owned until shutdown stops them. -export async function detachIosRunnerSessionsForShutdown(): Promise { - return await detachRunnerSessionsForShutdown(runnerSessions, (deviceId) => { - runnerSessions.delete(deviceId); +export async function detachIosRunnerSessionsForShutdown(): Promise { + return await detachRunnerSessionsForShutdown(runnerSessions(), (deviceId) => { + runnerSessions().delete(deviceId); }); } -export async function stopAllIosRunnerSessions(): Promise { +/** Stops every remaining runner and names the devices it owned. */ +export async function stopAllIosRunnerSessions(): Promise { + const owned = [...runnerSessions().keys()]; // This is the daemon's teardown: the fence spans the whole sweep, not just the abort, so a // start that began while the per-session stops ran meets a closed admission at its next gate // instead of answering the final prep sweep with another build (#3220). - const settleFences = retireAllRunnerStartAdmissions(runnerSessions.keys()); + const settleFences = retireAllRunnerStartAdmissions(runnerSessions().keys()); try { await abortAllIosRunnerSessions(); - const pending = Array.from(runnerSessions.keys()); + const pending = Array.from(runnerSessions().keys()); await Promise.allSettled( pending.map(async (deviceId) => { await stopIosRunnerSession(deviceId); }), ); await stopRunnerPrepProcesses(); + return owned; } finally { settleFences(); } @@ -852,7 +862,7 @@ export async function executeRunnerCommandWithSession( (reason) => invalidateRunnerSession(session, reason), signal, ); - if (runnerSessions.get(device.id) === session && session.state === 'ready') { + if (runnerSessions().get(device.id) === session && session.state === 'ready') { const { observeRunnerListener } = await import('./runner-retention.ts'); observeRunnerListener(session, () => { const retention = session.retention; diff --git a/src/__tests__/test-utils/public-daemon-runtime.ts b/src/__tests__/test-utils/public-daemon-runtime.ts index ad8c87897c..6e8c3fcf02 100644 --- a/src/__tests__/test-utils/public-daemon-runtime.ts +++ b/src/__tests__/test-utils/public-daemon-runtime.ts @@ -52,8 +52,6 @@ export async function startPublicDaemonRuntime( AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -74,12 +72,12 @@ export async function startPublicDaemonRuntime( baseUrl: `http://127.0.0.1:${port}`, runtime, close: async () => { - await runtime.shutdown(); + await runtime.close(); fs.rmSync(stateDir, { recursive: true, force: true }); }, }; } catch (error) { - await runtime.shutdown().catch(() => {}); + await runtime.close().catch(() => {}); if (ownsStateDir) fs.rmSync(stateDir, { recursive: true, force: true }); throw error; } diff --git a/src/daemon-composition-credentials.test.ts b/src/daemon-composition-credentials.test.ts index 820fdc081d..348a278a69 100644 --- a/src/daemon-composition-credentials.test.ts +++ b/src/daemon-composition-credentials.test.ts @@ -14,8 +14,8 @@ vi.mock('./platform-runtime.ts', () => ({ createPlatformRuntimeGateway: () => ({ applicationLifecycle: { recoverStartupResources: async () => {}, - detachForDaemonShutdown: async () => {}, - finalizeDaemonShutdown: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], }, inspectFacts: async () => { throw new Error('unused'); @@ -46,8 +46,6 @@ test('daemon startup compares lease credentials with its own startup environment AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: createDaemonRuntimeDeps, @@ -81,6 +79,6 @@ test('daemon startup compares lease credentials with its own startup environment ); expect(await allocate(daemonEnv)).not.toContain('provider-credentials-changed'); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); diff --git a/src/daemon-composition.test.ts b/src/daemon-composition.test.ts index db7617016b..97d10b6fe2 100644 --- a/src/daemon-composition.test.ts +++ b/src/daemon-composition.test.ts @@ -17,8 +17,8 @@ vi.mock('./platform-runtime.ts', () => ({ createPlatformRuntimeGateway: () => ({ applicationLifecycle: { recoverStartupResources: async () => {}, - detachForDaemonShutdown: async () => {}, - finalizeDaemonShutdown: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], }, inspectFacts: async () => { throw new Error('unused'); diff --git a/src/daemon-composition.ts b/src/daemon-composition.ts index 773763d24b..ec736f40b5 100644 --- a/src/daemon-composition.ts +++ b/src/daemon-composition.ts @@ -125,7 +125,11 @@ export const createDaemonRuntimeDeps: ComposeDaemonRuntimeDeps = async ( const { createDaemonPlatformServices } = await import('./platform-runtime-daemon-services.ts'); const { createClaimRecoveryGateway } = await import('./platform-runtime-claim-recovery.ts'); + const { createAppleRunnerRuntimeState } = + await import('@agent-device/platform-apple/runner-owner'); + return Object.freeze({ + runtimeStates: Object.freeze([createAppleRunnerRuntimeState()]), ownedProcessRecords, hostDiagnostics, ownerLifecycle: platformDaemonLifecycleOwners, diff --git a/src/daemon-entrypoint.test.ts b/src/daemon-entrypoint.test.ts index e28372b631..1379348e69 100644 --- a/src/daemon-entrypoint.test.ts +++ b/src/daemon-entrypoint.test.ts @@ -78,7 +78,6 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a const paths = resolveDaemonPaths(stateDir); const stdout: string[] = []; const stderr: string[] = []; - let exitCode: number | undefined; try { const runtime = await startDaemonRuntime({ @@ -90,10 +89,6 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: (chunk) => stderr.push(chunk) }, stdout: { write: (chunk) => stdout.push(chunk) }, }); @@ -111,9 +106,7 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a assert.ok(fs.existsSync(paths.lockPath), 'daemon lock should be held while runtime is active'); await waitForHttpOk(`http://127.0.0.1:${runtime?.httpPort}/health`, 2_000); - await runtime?.shutdown(); - - assert.equal(exitCode, 0); + await runtime?.close(); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -125,7 +118,6 @@ test('daemon runtime publishes dual transport metadata', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-runtime-dual-'); const paths = resolveDaemonPaths(stateDir); const stdout: string[] = []; - let exitCode: number | undefined; try { const runtime = await startDaemonRuntime({ @@ -137,10 +129,6 @@ test('daemon runtime publishes dual transport metadata', async () => { AGENT_DEVICE_DAEMON_SERVER_MODE: 'dual', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: (chunk) => stdout.push(chunk) }, }); @@ -156,9 +144,7 @@ test('daemon runtime publishes dual transport metadata', async () => { assert.equal(info.transport, 'dual'); await waitForHttpOk(`http://127.0.0.1:${runtime?.httpPort}/health`, 2_000); - await runtime?.shutdown(); - - assert.equal(exitCode, 0); + await runtime?.close(); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -193,8 +179,6 @@ test('daemon rejects unowned cloud artifacts over RPC', async () => { BROWSERSTACK_SESSION_DETAILS_ENDPOINT: `http://127.0.0.1:${providerPort}/sessions`, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -226,7 +210,7 @@ test('daemon rejects unowned cloud artifacts over RPC', async () => { assert.deepEqual(body.error?.data?.details, { reason: 'PROVIDER_SESSION_NOT_OWNED' }); assert.deepEqual(providerRequests, []); } finally { - await runtime?.shutdown(); + await runtime?.close(); await closeLoopbackServer(providerServer); fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -290,7 +274,6 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', const claimsDir = mkdtempForTestSync('agent-device-daemon-prune-claims-'); const paths = resolveDaemonPaths(stateDir); const deviceKey = 'local:android:none:prune-log'; - let exitCode: number | undefined; try { fs.writeFileSync( @@ -320,10 +303,6 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', AGENT_DEVICE_CLAIMS_DIR: claimsDir, }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -335,8 +314,7 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', assert.match(log, /"phase":"device_claim_reconcile"/); assert.match(log, /"reconciled":1/); - await runtime?.shutdown(); - assert.equal(exitCode, 0); + await runtime?.close(); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); fs.rmSync(claimsDir, { recursive: true, force: true }); @@ -444,8 +422,6 @@ test('startup sweep settles a foreign dead owner without touching a live same-na AGENT_DEVICE_CLAIMS_DIR: claimsDir, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -474,7 +450,7 @@ test('startup sweep settles a foreign dead owner without touching a live same-na "the dead owner's orphaned recorder should be terminated by recovery", ); } finally { - await runtime?.shutdown().catch(() => {}); + await runtime?.close().catch(() => {}); try { orphanRecorder.kill('SIGKILL'); } catch {} diff --git a/src/daemon-runtime-limrun-public-http.test.ts b/src/daemon-runtime-limrun-public-http.test.ts index 9ee0387c04..98ccbf07cc 100644 --- a/src/daemon-runtime-limrun-public-http.test.ts +++ b/src/daemon-runtime-limrun-public-http.test.ts @@ -19,8 +19,8 @@ const limrunIo = vi.hoisted(() => ({ const rootPorts = vi.hoisted(() => { const applicationLifecycle = { recoverStartupResources: vi.fn(async () => {}), - detachForDaemonShutdown: vi.fn(async () => {}), - finalizeDaemonShutdown: vi.fn(async () => {}), + detachForDaemonShutdown: vi.fn(async () => []), + finalizeDaemonShutdown: vi.fn(async () => []), }; return { providerComposition: vi.fn(async () => ({ runtimes: [] as unknown[], platformModules: [] })), @@ -93,8 +93,6 @@ test('public HTTP rejects Limrun uploaded-app listing and allocation before prov LIM_ANDROID_INSTANCE_ADB_URL: 'https://region.limrun.example/v1/android_test/adb', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -149,7 +147,7 @@ test('public HTTP rejects Limrun uploaded-app listing and allocation before prov assert.equal(rootPorts.createRequestPlatformProviders.mock.calls.length, 1); assert.equal(rootPorts.applicationLifecycle.recoverStartupResources.mock.calls.length, 1); } finally { - await runtime?.shutdown(); + await runtime?.close(); await limrun.shutdown(); fs.rmSync(paths.baseDir, { recursive: true, force: true }); if (runtime) diff --git a/src/daemon.ts b/src/daemon.ts index 264b1de596..d5ad7b8c64 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -1,16 +1,43 @@ import { asAppError } from '@agent-device/kernel/errors'; -import { startDaemonRuntime } from '@agent-device/daemon'; +import { daemonStartupExitCode, startDaemonRuntime } from '@agent-device/daemon'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; +// The executable owns the process: signals, fatal-error handlers and the exit code. The runtime +// only reports when it has closed, whoever closed it. // The root's composition enters as a factory behind a function-scoped `await import`: the server // calls it before it acquires the registration, so startup order is unchanged, and the eager // closure of this entry stays exactly the operational server's (ADR 0019). -void startDaemonRuntime({ - host: captureHostEnvironment(process), - composeDeps: async (input) => - (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input), -}).catch((error) => { - const appErr = asAppError(error); - process.stderr.write(`Daemon error: ${appErr.message}\n`); - process.exit(1); +async function runDaemon(): Promise { + const runtime = await startDaemonRuntime({ + host: captureHostEnvironment(process), + composeDeps: async (input) => + (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input), + }); + let exitCode = 0; + const closeAndExit = (code: number, cause?: unknown): void => { + exitCode = Math.max(exitCode, code); + // A failed close is reported once, through `closed` below. + void runtime.close({ cause }).catch(() => {}); + }; + const failFatally = (error: unknown): void => { + process.stderr.write(`Daemon error: ${asAppError(error).message}\n`); + closeAndExit(1, error); + }; + process.on('SIGINT', () => closeAndExit(0)); + process.on('SIGTERM', () => closeAndExit(0)); + process.on('SIGHUP', () => closeAndExit(0)); + process.on('uncaughtException', failFatally); + process.on('unhandledRejection', (reason) => + failFatally(reason instanceof Error ? reason : new Error(String(reason))), + ); + await runtime.closed.catch((error: unknown) => { + process.stderr.write(`Daemon shutdown error: ${asAppError(error).message}\n`); + exitCode = 1; + }); + process.exit(exitCode); +} + +runDaemon().catch((error) => { + process.stderr.write(`Daemon error: ${asAppError(error).message}\n`); + process.exit(daemonStartupExitCode(error)); }); diff --git a/src/platform-runtime-apple-application-tools.ts b/src/platform-runtime-apple-application-tools.ts index 35985fe3b0..b546344a86 100644 --- a/src/platform-runtime-apple-application-tools.ts +++ b/src/platform-runtime-apple-application-tools.ts @@ -114,11 +114,11 @@ export function createAppleApplicationTools(): AppleApplicationTools { dismissCloseAlerts: async (device, input) => await dismissMacOsCloseAlerts(device, input), detachRunnerSessionsForShutdown: async () => { const { detachIosRunnerSessionsForShutdown } = await loadRunnerOperations(); - await detachIosRunnerSessionsForShutdown(); + return await detachIosRunnerSessionsForShutdown(); }, finalizeRunnerSessionsForShutdown: async () => { const { stopAllIosRunnerSessions } = await loadRunnerOperations(); - await stopAllIosRunnerSessions(); + return await stopAllIosRunnerSessions(); }, }); } diff --git a/src/platform-runtime-application-resources.test.ts b/src/platform-runtime-application-resources.test.ts index 4ad3eee04e..ea728b5cdf 100644 --- a/src/platform-runtime-application-resources.test.ts +++ b/src/platform-runtime-application-resources.test.ts @@ -43,8 +43,8 @@ test('startup recovery reaches Android mechanics only after validated marker evi }); test('daemon shutdown phases delegate to the Apple runner owner', async () => { - const detachRunnerSessionsForShutdown = vi.fn(async () => {}); - const finalizeRunnerSessionsForShutdown = vi.fn(async () => {}); + const detachRunnerSessionsForShutdown = vi.fn(async () => ['ios-sim']); + const finalizeRunnerSessionsForShutdown = vi.fn(async () => ['ios-sim']); const resources = lifecycle({ hasTestImeRecoveryEvidence: async () => false, recoverTestImeStartup: async () => {}, diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index 54feedf435..867cb313ea 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -74,8 +74,8 @@ export function createComposedPlatformRuntimeGateway(options: { }; // Which durable resources exist is the host's composition problem; the gateway only owns the // once-per-process shape of each phase and keeps the host load lazy. - let lifecycleDetach: Promise | undefined; - let lifecycleFinalize: Promise | undefined; + let lifecycleDetach: Promise | undefined; + let lifecycleFinalize: Promise | undefined; const applicationLifecycle = Object.freeze({ recoverStartupResources: async (input: Readonly<{ stateDir: string }>) => { await runStartupRecoveryFence(input.stateDir, async () => { @@ -86,13 +86,13 @@ export function createComposedPlatformRuntimeGateway(options: { lifecycleDetach ??= loadHost().then( async (host) => await host.applicationResources.detachForDaemonShutdown(), ); - await lifecycleDetach; + return await lifecycleDetach; }, finalizeDaemonShutdown: async () => { lifecycleFinalize ??= loadHost().then( async (host) => await host.applicationResources.finalizeDaemonShutdown(), ); - await lifecycleFinalize; + return await lifecycleFinalize; }, }); const registerOwner = (owner: PlatformRuntimeOwner) => { diff --git a/src/provider-device-runtimes.test.ts b/src/provider-device-runtimes.test.ts index 8ef20ecdb5..7269651fd5 100644 --- a/src/provider-device-runtimes.test.ts +++ b/src/provider-device-runtimes.test.ts @@ -73,7 +73,7 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest interval: 10, timeout: 2_000, }); - await firstDaemon.runtime.shutdown(); + await firstDaemon.runtime.close(); assert.ok(limrun.iosDisconnect.mock.calls.length > 0); const pendingPath = path.join(stateDir, 'expired-provider-leases.json'); @@ -100,7 +100,7 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest if (restartedDaemon) { await restartedDaemon.close(); } else { - await firstDaemon?.runtime.shutdown(); + await firstDaemon?.runtime.close(); fs.rmSync(stateDir, { recursive: true, force: true }); } }