diff --git a/Dockerfile b/Dockerfile index 4822907..8e2d8a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ FROM node:22-bookworm-slim # git + gh are the loop's ground-truth controllers; tmux is required for context # autophagy (the pane is the respawn mutex — LLP 0013). RUN apt-get update && apt-get install -y --no-install-recommends \ - git tmux curl ca-certificates jq procps util-linux \ + git tmux curl ca-certificates jq procps util-linux tini \ && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ diff --git a/docker/SAFETY.md b/docker/SAFETY.md index a24547f..8bd3d3f 100644 --- a/docker/SAFETY.md +++ b/docker/SAFETY.md @@ -5,11 +5,13 @@ its second failure within a rolling 60 minutes. Six failures within 24 hours also hold it. Each loop may start at most four times per hour and 24 times per day, including context recycling. These are restart limits, not token quotas. -The root PID 1 controller owns admission and durable state. Claude, HypAware, -the bridge, and sentinel run as `neutral`. A trip persists the hold and exits -PID 1, terminating detached workers with the container. Docker may restart the -container, but it then runs diagnostics only. The operator must explicitly -rearm it. See [LLP 0072](../llp/0072-restart-burn-safeguard.rfc.md). +The root controller owns admission and durable state, as the direct child of +the bundled tini PID 1. Tini reaps orphaned workers after they exit. Claude, +HypAware, the bridge, and sentinel run as `neutral`. A trip persists the hold +and exits the controller; tini then exits, terminating detached workers with +the container. Docker may restart the container, but it then runs diagnostics only. The operator must explicitly +rearm it. See [LLP 0072](../llp/0072-restart-burn-safeguard.rfc.md) and its +[process-reaping extension](../llp/0076-reap-orphaned-workers.spec.md). ## Deployment and first boot @@ -29,8 +31,9 @@ leave the replacement held after initialization, not to rearm repeatedly. Keep that volume for the lifetime of the fleet, including image upgrades, container recreation, and moves to another host. The image now starts as root; do not override its user, entrypoint, PID namespace, or enable Docker's `--init` -(the controller must be PID 1). An exclusive file lock prevents two containers -from running against the same safety volume. Agents receive no sudo access or +(the image already provides tini; the controller must be its direct child). +An exclusive file lock prevents two containers from running against the same +safety volume. Agents receive no sudo access or Docker socket. Do not run an older, unguarded rollback image against these volumes and assume that it understands the hold. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 77d979c..0a83a1c 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -# @ref LLP 0072#controller [implements] — root PID 1, exclusive persistent ownership, no model launch before admission +# @ref LLP 0076#init [implements] - tini reaps orphans and exits with the admission owner set -euo pipefail -[ "$(id -u)" = 0 ] || { echo 'neutral safety entrypoint requires root' >&2; exit 75; } +[ "$(id -u)" = 0 ] && [ "$$" = 1 ] || { echo 'neutral safety entrypoint requires root PID 1 (no external --init)' >&2; exit 75; } mkdir -p /var/lib/neutral-safety chown root:root /var/lib/neutral-safety chmod 700 /var/lib/neutral-safety -# --no-fork makes Node PID 1. Its exit kills the whole PID namespace, including -# detached model workers. Child launches close the inherited lock descriptor. -exec flock --nonblock --no-fork /var/lib/neutral-safety/owner.lock node /opt/neutral/docker/safety-controller.js +# flock execs Node without forking: tini's direct child owns the lock. When +# that child exits, tini exits too, terminating every remaining descendant. +exec /usr/bin/tini -- flock --nonblock --no-fork /var/lib/neutral-safety/owner.lock node /opt/neutral/docker/safety-controller.js diff --git a/docker/prepare.sh b/docker/prepare.sh index 80c5252..9e4d3b2 100755 --- a/docker/prepare.sh +++ b/docker/prepare.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# @ref LLP 0072#controller [implements] — unprivileged setup only; PID 1 owns every process launch +# @ref LLP 0072#controller [implements] — unprivileged setup only; the controller owns every process launch set -euo pipefail log() { printf '[neutral-prepare] %s\n' "$*"; } [ "$(id -u)" != 0 ] || { log 'setup must run as neutral'; exit 1; } diff --git a/docker/safety-controller.js b/docker/safety-controller.js index 6d68aaa..14c58a7 100644 --- a/docker/safety-controller.js +++ b/docker/safety-controller.js @@ -1,7 +1,7 @@ // @ts-check import { randomUUID } from 'node:crypto' import { createServer } from 'node:net' -import { chmodSync, mkdirSync, readFileSync, existsSync, rmSync } from 'node:fs' +import { chmodSync, mkdirSync, readFileSync, existsSync, rmSync, readlinkSync } from 'node:fs' import { pathToFileURL } from 'node:url' import { setTimeout as delay } from 'node:timers/promises' import { initialSafetyState, safetyStep, safetyCounts, SAFETY_POLICY } from '../src/safety.js' @@ -292,7 +292,7 @@ export function serveSafety(controller, path, operator) { /** * Docker restart preserves the writable layer, including dead Unix sockets. - * Called only by PID 1 after entrypoint's exclusive persistent lock, before + * Called only by the root controller after entrypoint's exclusive persistent lock, before * starting any service. Never remove the persistent state directory here. * @ref LLP 0072#validation [implements] — diagnostics must survive Docker restart, not only recreation * @param {string} [dir] @@ -303,7 +303,10 @@ export function prepareSafetyRuntime(dir = RUNTIME_DIR) { } async function main() { - if (process.getuid?.() !== 0 || process.pid !== 1) throw new Error('safety controller must run as root PID 1') + // @ref LLP 0076#init [implements] - refuse an unmanaged controller whose exit cannot end the namespace + if (process.getuid?.() !== 0 || process.ppid !== 1 || readlinkSync('/proc/1/exe') !== '/usr/bin/tini') { + throw new Error('safety controller must run as the direct root child of the bundled tini PID 1') + } // No fallback to the disposable container layer: a missing mount is held. const mounted = readFileSync('/proc/self/mountinfo', 'utf8').split('\n').some(l => l.split(' ')[4] === SAFETY_DIR) prepareSafetyRuntime() diff --git a/docker/safety-store.js b/docker/safety-store.js index 1772711..17c09e9 100644 --- a/docker/safety-store.js +++ b/docker/safety-store.js @@ -22,7 +22,7 @@ export function validSafetyState(value) { Array.isArray(s.starts) && s.starts.length <= 24 * 100 && s.starts.every(e => str(e.id) && str(e.loop) && num(e.at) && e.at <= s.clock.mono) } -// @ref LLP 0072#persistence [implements] — atomic checkpoint + bounded synced audit, owned by PID 1 +// @ref LLP 0072#persistence [implements] — atomic checkpoint + bounded synced audit, owned by the root controller export class SafetyStore { /** @param {string} dir */ constructor(dir) { this.dir = dir } diff --git a/docker/test-fixtures/safety/Dockerfile b/docker/test-fixtures/safety/Dockerfile index e132924..70584d4 100644 --- a/docker/test-fixtures/safety/Dockerfile +++ b/docker/test-fixtures/safety/Dockerfile @@ -1,7 +1,7 @@ ARG BASE_IMAGE=node:22-bookworm-slim FROM ${BASE_IMAGE} USER root -RUN if ! command -v tmux >/dev/null || ! command -v setpriv >/dev/null; then apt-get update && apt-get install -y --no-install-recommends tmux util-linux && rm -rf /var/lib/apt/lists/*; fi +RUN if ! command -v tmux >/dev/null || ! command -v setpriv >/dev/null || ! command -v tini >/dev/null; then apt-get update && apt-get install -y --no-install-recommends tmux util-linux tini && rm -rf /var/lib/apt/lists/*; fi RUN (id neutral >/dev/null 2>&1 || useradd -m -s /bin/bash neutral) && mkdir -p /work/a /var/lib/neutral-safety && chown -R neutral:neutral /work && chmod 700 /var/lib/neutral-safety COPY . /opt/neutral RUN chown -R root:root /opt/neutral && chmod -R go-w /opt/neutral && \ diff --git a/llp/0072-restart-burn-safeguard.rfc.md b/llp/0072-restart-burn-safeguard.rfc.md index a37365d..4d61ff6 100644 --- a/llp/0072-restart-burn-safeguard.rfc.md +++ b/llp/0072-restart-burn-safeguard.rfc.md @@ -5,6 +5,7 @@ **Systems:** Engine **Author:** Codex **Date:** 2026-09-07 +**Extended-by:** LLP 0076 (PID 1 packaging and orphan reaping) **Related:** 0001, 0002, 0010, 0013, 0015, 0034, 0039, 0057 ## Recommendation diff --git a/llp/0076-reap-orphaned-workers.spec.md b/llp/0076-reap-orphaned-workers.spec.md new file mode 100644 index 0000000..834b011 --- /dev/null +++ b/llp/0076-reap-orphaned-workers.spec.md @@ -0,0 +1,44 @@ +# LLP 0076: Reap orphaned workers without weakening the fleet safety hold + +**Type:** Spec +**Status:** Accepted +**Systems:** Engine +**Author:** Codex +**Date:** 2026-09-11 +**Related:** 0072 + +@ref LLP 0072#controller [extends] - change PID 1 packaging while preserving one root admission owner +@ref LLP 0072#recovery [constrained-by] - controller death still terminates the entire PID namespace + +## Evidence + +The September 10 deployment put Node directly at PID 1. The next day the +container had 3,057 zombies, all adopted by that PID. Node collects children +it spawned through its own handles, but does not reap arbitrary orphaned +worker descendants. An isolated shell-grandchild reproduction leaves a +zombie under Node PID 1 and none under tini. + + +## Required process boundary + +Package tini as PID 1, with the root Node safety controller as its direct +child. The entrypoint must reject an external init or shared PID namespace. +The exclusive flock must exec the controller without another parent process. +The controller must reject startup without the bundled tini as its PID 1 +parent. Tini forwards termination signals, reaps adopted children, and exits +when its direct child exits. Its exit ends the namespace, killing detached +workers even after a controller crash or failed hold write. + +This supersedes only the requirement that Node itself be PID 1. Admission, +operator permissions, persistence, restart limits, and held-boot behavior +remain as specified in LLP 0072. Installing the fix does not authorize rearming +or restarting a fleet an operator stopped. + + +## Verification + +The fake-service Docker smoke must observe that exited orphaned grandchildren +disappear from /proc while the controller stays alive. It must also verify +controller death kills detached workers and reboots held, and retain the +existing recovery, durable hold, and unauthorized-operation checks. A direct +unmanaged controller and an externally supplied --init must fail closed. diff --git a/scripts/safety-smoke.mjs b/scripts/safety-smoke.mjs index cff298d..5678385 100644 --- a/scripts/safety-smoke.mjs +++ b/scripts/safety-smoke.mjs @@ -36,7 +36,11 @@ function start() { } const modelStarts = () => JSON.parse(exec('node', '-e', "const f=require('fs'); console.log(JSON.stringify(f.existsSync('/work/model-starts.jsonl')?f.readFileSync('/work/model-starts.jsonl','utf8').trim().split('\\n').map(JSON.parse):[]))")) const workers = () => exec('node', '-e', "const f=require('fs'); console.log(JSON.stringify(Object.fromEntries(f.readdirSync('/work').filter(p=>p.startsWith('worker-')).map(p=>[p,f.readFileSync('/work/'+p,'utf8')]))))") +// @ref LLP 0076#validation [tests] - reaping and controller death preserve the container boundary try { + assert.throws(() => docker('run', '--rm', '--entrypoint', 'node', image, '/opt/neutral/docker/safety-controller.js')) + assert.throws(() => docker('run', '--rm', '--init', image)) + mark('unmanaged-controller-and-external-init-refused') docker('run', '--rm', '--entrypoint', 'node', image, '/opt/neutral/docker/test-fixtures/safety/probe.js') mark('process-owned-gateway-only') docker('volume', 'create', safety) @@ -55,8 +59,21 @@ try { root('rearm', '--incident', initialized.hold.id, '--reason', 'first fake fleet boot') await until(() => modelStarts().length === 1) assert.equal(exec('id', '-u'), '0') - assert.equal(exec('node', '-e', "console.log(require('fs').readFileSync('/proc/1/comm','utf8').trim())"), 'node') + assert.equal(exec('node', '-e', "console.log(require('fs').readFileSync('/proc/1/comm','utf8').trim())"), 'tini') mark('operator-rearm-launches-after-stable-gateway') + // Exiting grandchildren are adopted by PID 1, outside Node's child handles. + const orphanPids = JSON.parse(exec('node', '-e', ` + const { spawnSync } = require('node:child_process') + const pids = Array.from({ length: 12 }, () => Number(spawnSync('sh', + ['-c', 'sleep 0.1 >/dev/null 2>&1 & echo $!'], { encoding: 'utf8' }).stdout.trim())) + console.log(JSON.stringify(pids)) + `)) + assert(orphanPids.every(pid => Number.isInteger(pid) && pid > 1)) + await until(() => JSON.parse(exec('node', '-e', ` + const fs = require('node:fs') + console.log(JSON.stringify(${JSON.stringify(orphanPids)}.every(pid => !fs.existsSync('/proc/' + pid)))) + `)), 10_000) + mark('orphaned-exited-children-are-reaped') let before = status() exec('kill', '-9', String(before.processes.find(p => p.role === 'hyp').pid)) await until(() => modelStarts().length === 2) @@ -87,6 +104,24 @@ try { assert.equal(audit.filter(e => e.event_type === 'failure').length, 2) assert(audit.some(e => e.reason === 'second failure within 60 minutes')) mark('durable-audit-matches-observed-launches') + const restarts = Number(docker('inspect', '--format', '{{.RestartCount}}', run)) + root('rearm', '--incident', incident, '--reason', 'verify controller crash terminates descendants') + await until(() => modelStarts().length === 3) + const controllerPid = exec('node', '-e', ` + const fs = require('node:fs') + const children = fs.readFileSync('/proc/1/task/1/children', 'utf8').trim().split(/\\s+/) + console.log(children.filter(pid => fs.readFileSync('/proc/' + pid + '/cmdline', 'utf8') + .includes('/opt/neutral/docker/safety-controller.js')).join(' ')) + `) + assert.match(controllerPid, /^\d+$/) + exec('kill', '-9', controllerPid) + await until(() => Number(docker('inspect', '--format', '{{.RestartCount}}', run)) > restarts) + await until(() => status().held) + assert.equal(modelStarts().length, 3) + const crashedWorkers = workers() + await delay(1500) + assert.equal(workers(), crashedWorkers) + mark('controller-crash-kills-descendants-and-reboots-held') console.log(`Smoke evidence: ${output}`) } finally { try { writeFileSync(join(output, 'container.log'), docker('logs', '--tail', '100', run)) } catch {}