CI: stop leaking test temp dirs and orphaned daemons on the Linux runner

.github/workflows/windows.yml

@@ -274,6 +274,28 @@ jobs:
           "CARGO_TARGET_DIR=$dir" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
           Write-Host "cargo target dir: $dir"
 
+      # Job-scoped temp root for the Linux fleet leg. The box's /tmp is a
+      # RAM-backed tmpfs shared by both listeners, the operator and every
+      # earlier job; test and smoke scratch that escaped cleanup (and the
+      # daemons a cancelled job orphaned) piled up there for months —
+      # ~60k dirs and over 40% of its inodes by 2026-10. Every later step's
+      # tempfile/mkdtemp/os.tmpdir() now lands under this listener's
+      # $RUNNER_TEMP instead: the runner wipes that dir at every job start
+      # and end, and the always() janitor at the end of this job removes
+      # it after stopping the processes that still reference it. Kept well
+      # under macOS's TMPDIR length, so Unix-socket paths built under it
+      # stay inside sun_path. Linux fleet only: hosted runners are
+      # ephemeral, and the Mac's job hooks already reap stale temp.
+      - name: Job-scoped temp dir (linux fleet)
+        if: runner.environment != 'github-hosted' && runner.os == 'Linux'
+        shell: bash
+        run: |
+          job_tmp="$RUNNER_TEMP/tmp"
+          rm -rf "$job_tmp"
+          mkdir -m 700 "$job_tmp"
+          echo "TMPDIR=$job_tmp" >> "$GITHUB_ENV"
+          echo "job temp dir: $job_tmp"
+
       # rustup ships preinstalled on the GitHub-hosted runners; just pin the
       # target this job type-checks against.
       - name: Install Rust target
@@ -556,6 +578,10 @@ jobs:
           "$RUNNER_TEMP/pyws/bin/pip" install --quiet websockets
           echo "$RUNNER_TEMP/pyws/bin" >> "$GITHUB_PATH"
 
+      - name: CI job-scope janitor tests
+        if: runner.os == 'Linux' && steps.relevance_unix.outputs.relevant != 'false'
+        run: python3 -m unittest scripts/ci/test_reap_job_scope.py
+
       - name: MCP terminal continuity regressions
         if: runner.os == 'Linux' && steps.relevance_unix.outputs.relevant != 'false'
         run: |
@@ -717,10 +743,15 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
+      # Waits for the exit: on the Linux fleet the job's final janitor
+      # would otherwise catch a daemon still shutting down and report it
+      # as a leak.
       - name: Stop daemon (dashboard-boot)
         if: always() && steps.daemon.outputs.pid != ''
         shell: bash
-        run: kill "${{ steps.daemon.outputs.pid }}" 2>/dev/null || true
+        run: |
+          kill "${{ steps.daemon.outputs.pid }}" 2>/dev/null || true
+          timeout 15 tail --pid="${{ steps.daemon.outputs.pid }}" -f /dev/null || true
 
       # Inline visibility first (the artifact needs a download), then the
       # full daemon log + the smoke's screenshot/error dumps as artifacts.
@@ -740,3 +771,22 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
           overwrite: true
+
+      # Last step, on success, failure AND cancellation: stop whatever this
+      # job left running and delete the job temp dir. A cancelled job's
+      # tree kill SIGKILLs test binaries before their kill_on_drop cleanup
+      # runs, so the daemons they spawned survive, reparented to init (one
+      # ran for 23 days); scrubbed-env acceptance daemons are invisible to
+      # an env-marker sweep. All of them still reference the job temp dir
+      # (the inherited TMPDIR, or a HOME/cwd/profile minted under it),
+      # which lives under this listener's $RUNNER_TEMP — never the other
+      # listener's. scripts/ci/reap_job_scope.py has the rules, never fails
+      # the job, and warns per process it had to stop. Must stay after
+      # every step that uses a job process or the temp dir.
+      - name: Reap job-scoped processes and temp (linux fleet)
+        if: always() && runner.environment != 'github-hosted' && runner.os == 'Linux'
+        shell: bash
+        run: |
+          if [ -f scripts/ci/reap_job_scope.py ]; then
+            python3 scripts/ci/reap_job_scope.py --scope "$RUNNER_TEMP/tmp" --remove
+          fi

scripts/ci/README.md

@@ -651,12 +651,49 @@ connects time out". Empty port (default) disables it.
   of `com.intendant.ci.sccache` when the Linux hosts get the hooks
   treatment.
 
+## Linux job scope (in-workflow janitor)
+
+The Linux runner accounts have no job hooks yet, and the box's `/tmp`
+is a RAM-backed tmpfs shared by both listeners. Test scratch that
+escaped cleanup, plus the daemons cancelled jobs orphaned, accumulated
+there for months (by 2026-10: ~60k dirs, over 40% of its inodes, and
+three CI daemons up to 23 days old). Since then the Linux fleet leg of
+`windows.yml` scopes every job:
+
+- **"Job-scoped temp dir"** points `TMPDIR` at `$RUNNER_TEMP/tmp`
+  (per listener; the runner itself wipes `$RUNNER_TEMP` at every job
+  start and end).
+- **"Reap job-scoped processes and temp"** is the job's last step,
+  `always()` — so it also runs after a cancellation. `reap_job_scope.py`
+  stops this account's processes that still reference the job temp dir
+  (the inherited `TMPDIR`, or a cwd, argv entry or HOME minted under
+  it), sparing itself, its ancestors and the runner/sccache stack, then
+  deletes the dir. It never fails the job; each process it had to stop
+  shows up as a `::warning::` on the run — a leak signal to chase in the
+  test that spawned it.
+
+The process marker is the job temp dir rather than the process tree or
+`RUNNER_TRACKING_ID`: a cancelled job's tree kill SIGKILLs test
+binaries before their `kill_on_drop` cleanup runs (the children are
+reparented to init), and acceptance scripts launch daemons with a
+scrubbed environment. Residue from before this scoping (pre-2026-10
+`/tmp` dirs and daemons) is not matched and needs a one-time manual
+cleanup. Tests: `python3 -m unittest scripts/ci/test_reap_job_scope.py`.
+
 ## Interlocks
 
 - The workflow cache steps and this watchdog share the cache layout
   contract: per-listener key dirs with a `.last-used` marker. Change
   one, change both (`windows.yml` "External cargo target dir" steps ↔
   `fleet-watchdog.sh` prune/evict).
+- `windows.yml`'s "Job-scoped temp dir" and "Reap job-scoped processes
+  and temp" steps share the `$RUNNER_TEMP/tmp` layout with
+  `reap_job_scope.py`'s scope guard (`--scope` must be a dir directly
+  inside the runner's `<work>/_temp`). The reap step must stay the
+  job's last step: it stops anything still referencing that dir. A
+  server shared across jobs (an on-demand sccache, should the Linux
+  accounts adopt one) must be started with a TMPDIR outside the job
+  dir: the janitor spares sccache by name but still deletes the dir.
 - The in-job preflight floor (20G) is deliberately BELOW `STOP_GB`:
   the watchdog should pause listeners long before any job can see a
   sub-floor disk, leaving the preflight as the backstop for the

scripts/ci/reap_job_scope.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""Job-scope janitor for the self-hosted Linux CI leg (windows.yml).
+
+The Linux fleet leg points TMPDIR at a job-scoped directory,
+$RUNNER_TEMP/tmp ("Job-scoped temp dir" step), so every later step's
+processes inherit it and every tempfile/mkdtemp/os.tmpdir() root lands in
+it. This script runs as the job's last, `always()` step and:
+
+  1. stops every process of this account that still references the job
+     temp dir -- through its working directory, its argv or its
+     environment (the inherited TMPDIR, a HOME or profile dir minted under
+     it) -- except itself, its ancestors (the step shell, Runner.Worker),
+     its own children and the runner service stack: TERM, a short grace,
+     then KILL;
+  2. with --remove, deletes the job temp dir.
+
+Why a marker instead of the process tree: when a job is cancelled, the
+runner's tree kill SIGKILLs test binaries before their Drop-based
+`kill_on_drop` cleanup can run, so the test daemons they spawned are
+reparented to init and survive the job; acceptance scripts also launch
+daemons with a scrubbed environment, which hides them from any
+RUNNER_TRACKING_ID-based sweep. Every such process still references the
+job temp dir. It lives under the listener's own $RUNNER_TEMP and a
+listener runs one job at a time, so the marker never matches the other
+listener's live job.
+
+Always exits 0: janitorial trouble must never fail CI. Each reaped process
+is reported as a ::warning:: so leaks stay visible on the run.
+"""
+import argparse
+import os
+import re
+import shutil
+import signal
+import sys
+import time
+from pathlib import Path
+
+# Never signalled, whatever they reference: the runner service stack and a
+# shared compiler cache server (killing it mid-compile fails the other
+# listener's rustc invocations). Mirrors scripts/ci/hooks/hook-lib.sh.
+PROTECTED_MARKERS = (b'runsvc.sh', b'RunnerService.js', b'Runner.Listener',
+                     b'Runner.Worker', b'sccache')
+
+
+def valid_scope(scope):
+    """Accept only an absolute job temp dir inside a runner temp dir
+    (<work>/_temp/<name>).
+
+    An empty or shallow scope would match nearly every process of the
+    account, so anything else refuses the whole run.
+    """
+    path = Path(scope)
+    return (path.is_absolute() and os.path.normpath(scope) == scope
+            and path.parent.name == '_temp' and len(path.parts) >= 5)
+
+
+def references(value, root):
+    """True when `value` mentions `root` as a whole path: at the start, or
+    after `=`, `:`, whitespace or a quote (an env value, a PATH entry, a
+    `--flag=` argument), and followed by `/`, `:` or the end."""
+    pattern = rb'(?<![^=:\s\'"])' + re.escape(root) + rb'(?=/|:|$)'
+    return re.search(pattern, value) is not None
+
+
+def read_stat(proc_root, pid):
+    """(state, ppid, starttime) from /proc/<pid>/stat, or None if gone."""
+    try:
+        stat = (proc_root / str(pid) / 'stat').read_bytes()
+    except OSError:
+        return None
+    # comm may contain spaces and ')': the fields resume after the last ')'.
+    fields = stat[stat.rfind(b')') + 2:].split()
+    try:
+        return fields[0], int(fields[1]), int(fields[19])
+    except (IndexError, ValueError):
+        return None
+
+
+def real_uid(proc_root, pid):
+    try:
+        for line in (proc_root / str(pid) / 'status').read_bytes().splitlines():
+            if line.startswith(b'Uid:'):
+                return int(line.split()[1])
+    except (OSError, IndexError, ValueError):
+        pass
+    return None
+
+
+def read_list(proc_root, pid, name):
+    try:
+        return [item for item in (proc_root / str(pid) / name).read_bytes().split(b'\0') if item]
+    except OSError:
+        return []
+
+
+def scope_reason(proc_root, pid, root):
+    """Why `pid` belongs to the job scope, or None."""
+    try:
+        cwd = os.fsencode(os.readlink(proc_root / str(pid) / 'cwd'))
+    except OSError:
+        cwd = b''
+    if references(cwd, root):
+        return 'cwd'
+    if any(references(arg, root) for arg in read_list(proc_root, pid, 'cmdline')):
+        return 'argv'
+    for entry in read_list(proc_root, pid, 'environ'):
+        if references(entry, root):
+            return 'env ' + entry.split(b'=', 1)[0].decode(errors='replace')
+    return None
+
+
+def find_residue(proc_root, scope, uid, self_pid):
+    """[(pid, starttime, command, reason)] for this account's job residue."""
+    root = os.fsencode(scope)
+    snapshot = {}
+    for entry in proc_root.iterdir():
+        if entry.name.isdigit():
+            stat = read_stat(proc_root, int(entry.name))
+            if stat is not None:
+                snapshot[int(entry.name)] = stat
+    protected = set()
+    pid = self_pid
+    while pid > 0 and pid not in protected:
+        protected.add(pid)
+        pid = snapshot[pid][1] if pid in snapshot else 0
+    # This script's own children (none today) are part of the step, too.
+    own_tree = {self_pid}
+    changed = True
+    while changed:
+        changed = False
+        for pid, (_, ppid, _) in snapshot.items():
+            if ppid in own_tree and pid not in own_tree:
+                own_tree.add(pid)
+                changed = True
+    residue = []
+    for pid, (state, _, start) in sorted(snapshot.items()):
+        if pid in protected or pid in own_tree or state == b'Z':
+            continue
+        if real_uid(proc_root, pid) != uid:
+            continue
+        argv = read_list(proc_root, pid, 'cmdline')
+        if any(marker in arg for arg in argv for marker in PROTECTED_MARKERS):
+            continue
+        reason = scope_reason(proc_root, pid, root)
+        if reason:
+            command = b' '.join(argv)[:160].decode(errors='replace') or '?'
+            residue.append((pid, start, command, reason))
+    return residue
+
+
+def same_process(proc_root, pid, start):
+    stat = read_stat(proc_root, pid)
+    return stat is not None and stat[2] == start and stat[0] != b'Z'
+
+
+def reap(proc_root, residue, grace, kill=os.kill):
+    """TERM, wait up to `grace` seconds, KILL; returns pids that survived."""
+    live = []
+    for pid, start, _, _ in residue:
+        # Re-check identity right before signalling: never hit a recycled pid.
+        if same_process(proc_root, pid, start):
+            try:
+                kill(pid, signal.SIGTERM)
+                live.append((pid, start))
+            except OSError:
+                pass
+    deadline = time.monotonic() + grace
+    while live and time.monotonic() < deadline:
+        time.sleep(0.2)
+        live = [(pid, start) for pid, start in live if same_process(proc_root, pid, start)]
+    for pid, start in live:
+        if same_process(proc_root, pid, start):
+            try:
+                kill(pid, signal.SIGKILL)
+            except OSError:
+                pass
+    time.sleep(0.5 if live else 0)
+    return [pid for pid, start in live if same_process(proc_root, pid, start)]
+
+
+def remove_scope(scope):
+    """Delete the job temp dir; returns how many entries it held."""
+    target = Path(scope)
+    if not target.is_dir():
+        return 0
+    entries = sum(1 for _ in target.iterdir())
+    shutil.rmtree(target, ignore_errors=True)
+    if target.exists():
+        print(f'::warning::job-scope janitor: could not fully remove {annotation(scope)}')
+    return entries
+
+
+def annotation(text):
+    """Escape text for a workflow-command message (%, CR, LF)."""
+    return str(text).replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A')
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
+    parser.add_argument('--scope', required=True, help='the job temp dir ($RUNNER_TEMP/tmp)')
+    parser.add_argument('--remove', action='store_true', help='delete the job temp dir afterwards')
+    parser.add_argument('--grace', type=float, default=5.0, help='seconds between TERM and KILL')
+    parser.add_argument('--proc-root', type=Path, default=Path('/proc'), help=argparse.SUPPRESS)
+    args = parser.parse_args(argv)
+    if not valid_scope(args.scope):
+        print(f'::warning::job-scope janitor: refusing scope {annotation(repr(args.scope))}')
+        return 0
+    try:
+        residue = find_residue(args.proc_root, args.scope, os.getuid(), os.getpid())
+        for pid, _, command, reason in residue:
+            print(f'::warning::job-scope janitor: stopping leftover process {pid} ({annotation(reason)}): '
+                  f'{annotation(command)}')
+        survivors = reap(args.proc_root, residue, args.grace) if residue else []
+        for pid in survivors:
+            print(f'::warning::job-scope janitor: process {pid} survived SIGKILL')
+        removed = remove_scope(args.scope) if args.remove else 0
+        print(f'job-scope janitor: stopped {len(residue) - len(survivors)} leftover process(es); '
+              f'removed {removed} temp entr{"y" if removed == 1 else "ies"}')
+    except Exception as error:  # noqa: BLE001 -- janitorial: report, never fail the job
+        print(f'::warning::job-scope janitor failed: {annotation(error)}')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())