From dada7c16f5c9a3461e58aa16ef414c15fd32fafe Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 30 Aug 2026 06:13:25 +0000 Subject: [PATCH] root-ubuntu: give a box with no swap 2G of it Every box this script provisions was running with zero swap, so the kernel's only answer to a memory spike was the OOM killer -- and what it picks is whatever was biggest, which on a dev box is the build, the language server, or the editor somebody was working in. 2G does not make a small box a big one; it turns "the process died" into "that got slow for a moment". A swapfile rather than a partition: provider images arrive with the whole disk given to /, so there is no partition to make, and a file can be resized or removed on a live box. The step converges like the rest of the script. It only ever acts on a box with NO swap at all, so a machine with a swap partition or zram is left alone rather than gaining a second, forgotten swapfile; the fstab entry is matched on the path, so a re-run adds nothing and a hand-edited line survives. It declines where a plain swapfile is wrong (btrfs needs chattr +C and no compression) or dangerous (a swapfile on zfs can deadlock the box under exactly the pressure it was added to survive), inside a container, where the kernel and its swap belong to the host, and when the disk cannot spare the space -- a full / breaks things a memory spike never would have. vm.swappiness goes to 10 with it. The default of 60 treats swap as another tier of memory and pages out pages that are still in use, which is how swap earns its reputation; 10 keeps it as the safety net. Two details worth keeping: the mode goes on before the chown and on its own, because chained behind a failing chown the file would stay readable and mkswap would still accept it, and swap is every secret the machine has ever paged out. And a failed mkswap or swapon removes the file, so a later run cannot mistake 2G of dead disk for working swap. SWAP_SIZE=0 turns the whole thing off. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013TerE4nvNU3jvS51nRR6Pd --- root-ubuntu.sh | 155 +++++++++++++++++++++++++-- server.conf.example | 11 ++ test/root-ubuntu.test.ts | 220 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 377 insertions(+), 9 deletions(-) diff --git a/root-ubuntu.sh b/root-ubuntu.sh index 8486672..ff15690 100755 --- a/root-ubuntu.sh +++ b/root-ubuntu.sh @@ -30,13 +30,14 @@ # automatically and refreshed # 2. apt update/upgrade + unattended security updates # 3. ufw -# 4. dotfiles (.zsh*, .bash*, .ssh*, ...) from $DOTFILES_REPO, if you have one -# 5. oh-my-zsh + plugins, oh-my-tmux, irssi configs -# 6. mise (curl https://mise.run | sh) -# 7. moshcode (curl https://moshcode.sh/install.sh | sh) -# 8. a per-user ssh-agent as a systemd user service -# 9. motd from $MOTD_URL -# 10. nginx per-user pages, per-user dev apps, TLS +# 4. a 2G swapfile, if the box has no swap at all +# 5. dotfiles (.zsh*, .bash*, .ssh*, ...) from $DOTFILES_REPO, if you have one +# 6. oh-my-zsh + plugins, oh-my-tmux, irssi configs +# 7. mise (curl https://mise.run | sh) +# 8. moshcode (curl https://moshcode.sh/install.sh | sh) +# 9. a per-user ssh-agent as a systemd user service +# 10. motd from $MOTD_URL +# 11. nginx per-user pages, per-user dev apps, TLS # # Usage, as root: # ./root-ubuntu.sh # first run, or a refresh @@ -94,6 +95,9 @@ # # Env overrides: # SSH_PORT=22 port to open in ufw +# SWAP_SIZE=2G swapfile to create when the box has no swap (0 = never) +# SWAP_FILE=/swapfile where that file goes +# SWAPPINESS=10 vm.swappiness once there is swap to speak of # ASSUME_YES=1 don't prompt (defaults: $DEFAULT_GROUPS; no privkey copy) # DEFAULT_GROUPS=... groups an account lands in when --groups is not passed # (default sudo,admin). An unattended run never prompts, so @@ -2131,6 +2135,141 @@ fi try "enable ufw at boot" systemctl enable ufw ufw status verbose || true +# ------------------------------------------------------------------ swap --- +# +# A box with no swap has no slack. The kernel's only answer to a memory spike +# is the OOM killer, and what it picks is whatever was biggest -- on these +# boxes, the build, the language server, or the editor someone was working in. +# 2G of swap does not make a small box a big one; it turns "the process died" +# into "that got slow for a moment", which is the difference between losing an +# afternoon and noticing nothing. +# +# Deliberately a swap FILE and not a partition: provider images arrive with the +# whole disk given to /, so there is no partition to make, and a file can be +# resized or removed on a live box. +SWAP_SIZE="${SWAP_SIZE:-2G}" # 0 disables; the box keeps whatever it has +SWAP_FILE="${SWAP_FILE:-/swapfile}" +# 60 (the default) treats swap as another tier of memory and pages out things +# that are still being used. 10 keeps it as the safety net it is meant to be. +SWAPPINESS="${SWAPPINESS:-10}" + +# 2G / 2048M / 2 (G assumed) -> megabytes +_size_mb() { + local s="${1^^}" + [[ "$s" =~ ^([0-9]+)([GM]?)$ ]] || return 1 + case "${BASH_REMATCH[2]}" in + M) printf '%s' "${BASH_REMATCH[1]}" ;; + *) printf '%s' "$(( BASH_REMATCH[1] * 1024 ))" ;; + esac +} + +_swap_sysctl() { + write_if_changed /etc/sysctl.d/60-profullstack-swap.conf </dev/null || true + note "vm.swappiness=$SWAPPINESS" +} + +configure_swap() { + local file="$SWAP_FILE" dir want_mb avail_mb fstype virt name type size _rest + + if [[ -z "$SWAP_SIZE" || "$SWAP_SIZE" == 0 ]]; then + info "swap disabled (SWAP_SIZE=$SWAP_SIZE) -- leaving this box as it is" + return 0 + fi + + # Somebody else's swap counts. A swapfile stacked on top of a swap + # partition or a zram device is not extra safety, it is a file nobody + # remembers making. + if swapon --show=NAME --noheadings 2>/dev/null | grep -q .; then + while read -r name type size _rest; do + info "swap already active: $name ($type, $size)" + done < <(swapon --show=NAME,TYPE,SIZE --noheadings 2>/dev/null) + _swap_sysctl + return 0 + fi + + # Containers share the host kernel, and its swap is the host's business. + # swapon in here either fails outright or is refused by the cgroup after + # the file has already been written. + virt="$(systemd-detect-virt --container 2>/dev/null)" + if [[ -n "$virt" && "$virt" != none ]]; then + info "inside a $virt container -- swap belongs to the host" + return 0 + fi + + dir="$(dirname "$file")" + fstype="$(df --output=fstype "$dir" 2>/dev/null | tail -1)" + case "$fstype" in + btrfs) + warn "no swapfile: btrfs needs one built its own way (chattr +C, no compression, no snapshots)" + return 0 ;; + zfs) + warn "no swapfile: a swapfile on zfs can deadlock the box -- use a zvol" + return 0 ;; + esac + + want_mb="$(_size_mb "$SWAP_SIZE")" || { + warn "swap: cannot read SWAP_SIZE=$SWAP_SIZE (want something like 2G or 2048M)" + return 1 + } + + # Filling the root disk to buy memory headroom is a bad trade: a full / + # breaks things a memory spike would not have touched. + avail_mb="$(df -BM --output=avail "$dir" 2>/dev/null | tail -1 | tr -dc '0-9')" + if [[ -n "$avail_mb" ]] && (( avail_mb < want_mb + 2048 )); then + warn "swap: ${avail_mb}M free on $dir, need ${want_mb}M plus headroom -- skipping" + return 0 + fi + + [[ -e "$file" && ! -f "$file" ]] && { warn "swap: $file exists and is not a file"; return 1; } + + log "creating ${SWAP_SIZE} of swap at $file" + # fallocate is instant, but on some filesystems it leaves unwritten extents + # that mkswap then refuses. dd always works and is only slow the once, so + # it is both the fallback and the retry. + rm -f "$file" + fallocate -l "${want_mb}M" "$file" 2>/dev/null \ + || dd if=/dev/zero of="$file" bs=1M count="$want_mb" status=none \ + || { warn "swap: could not allocate $file"; rm -f "$file"; return 1; } + # World-readable swap is every secret the machine has ever paged out, so + # the mode goes on first and on its own -- chained behind a chown, one + # failure there would leave the file readable and mkswap would still be + # happy with it. + chmod 0600 "$file" + chown root:root "$file" + if ! mkswap "$file" >/dev/null 2>&1; then + # the unwritten-extents case: write the bytes for real, then try once more + dd if=/dev/zero of="$file" bs=1M count="$want_mb" status=none + chmod 0600 "$file" + if ! mkswap "$file" >/dev/null 2>&1; then + warn "swap: mkswap failed on $file" + rm -f "$file" + return 1 + fi + fi + swapon "$file" || { warn "swap: swapon failed on $file"; rm -f "$file"; return 1; } + note "${SWAP_SIZE} swap at $file" + + # ...and again after a reboot. Matched on the path, so an entry someone has + # since edited (different options, a different priority) is left alone. + if awk -v f="$file" '$1 == f && $3 == "swap" { found = 1 } END { exit !found }' /etc/fstab; then + info "fstab already brings $file up at boot" + else + printf '%-16s none swap sw 0 0\n' "$file" >>/etc/fstab \ + && note "fstab: $file" + fi + + _swap_sysctl + return 0 +} + +log "configuring swap" +try "swap" configure_swap + # ------------------------------------------------------------------ motd --- # Fetched into a cache file; login just cats it and kicks off a background @@ -4494,6 +4633,8 @@ info "open: $(ufw status | awk '/ALLOW/{printf "%s ", $1}')" info "mosh: $(command -v mosh-server >/dev/null && echo "$(mosh-server --version 2>&1 | head -1)" || echo 'MISSING')" info "tail: $(tailscale ip -4 2>/dev/null | head -1 || echo 'not joined to a tailnet')" info "motd: $([[ -s $MOTD_CACHE ]] && echo "cached ($(wc -l <"$MOTD_CACHE") lines)" || echo 'empty')" +info "ram: $(free -h | awk '/^Mem:/{printf "%s total, %s available", $2, $7}')" +info "swap: $(swapon --show=NAME,SIZE --noheadings 2>/dev/null | awk '{printf "%s (%s) ", $1, $2}' | grep . || echo 'NONE -- one memory spike from the OOM killer')" echo printf ' %-12s %-6s %-6s %-9s %-6s %-5s %s\n' USER OMZ MISE MOSHCODE TMUX APPS SHELL while read -r login; do diff --git a/server.conf.example b/server.conf.example index ee988b6..383d4d8 100644 --- a/server.conf.example +++ b/server.conf.example @@ -32,6 +32,17 @@ # will happily lock you out of the box you are provisioning. #SSH_PORT=22 +# A box with no swap has no slack: the kernel's only answer to a memory spike +# is the OOM killer, and what it picks is whatever was biggest. A swapfile is +# created only when the box has no swap at all, so a machine that already has +# a swap partition or zram is left exactly as it is. Set 0 to never make one. +#SWAP_SIZE=2G +#SWAP_FILE=/swapfile + +# 60, the kernel default, treats swap as another tier of memory and pages out +# things that are still being used. 10 keeps it as the safety net it is for. +#SWAPPINESS=10 + # ── accounts ───────────────────────────────────────────────────────────────── # Groups new accounts land in when nothing is passed on the command line. diff --git a/test/root-ubuntu.test.ts b/test/root-ubuntu.test.ts index f4eb5e7..86cfed7 100644 --- a/test/root-ubuntu.test.ts +++ b/test/root-ubuntu.test.ts @@ -1,6 +1,7 @@ import { execFileSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; +import { mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; @@ -577,3 +578,218 @@ describe('the groups subcommand', () => { expect(out).toContain('groups add alice docker'); }); }); + +describe('_size_mb', () => { + const mb = (s: string) => shell(['_size_mb'], `_size_mb ${s}`); + + it('reads the sizes a person would actually write', () => { + expect(mb('2G')).toBe('2048'); + expect(mb('2g')).toBe('2048'); + expect(mb('512M')).toBe('512'); + }); + + it('assumes gigabytes for a bare number, because nobody means 2MB of swap', () => { + expect(mb('4')).toBe('4096'); + }); + + it('refuses anything it cannot read rather than guessing', () => { + // The value ends up in `dd count=`, so a silent misread is a swapfile of + // the wrong size -- or, with an empty count, no swapfile at all. + expect(status(['_size_mb'], '_size_mb abc')).toBe(1); + expect(status(['_size_mb'], '_size_mb "2 G"')).toBe(1); + expect(status(['_size_mb'], '_size_mb ""')).toBe(1); + expect(status(['_size_mb'], '_size_mb 2GB')).toBe(1); + }); +}); + +describe('configure_swap', () => { + /** + * The function reaches for swapon, mkswap, df and systemd-detect-virt, and + * writes to /etc/fstab and /etc/sysctl.d. Those tools are stubbed and the + * two absolute paths are rewritten into a temp directory with `declare -f`, + * so what runs is the code in the file and the test still cannot touch the + * machine it runs on. The SWAP_* declarations are top-level rather than + * inside a function, so they are lifted out of the source the same way the + * DEFAULT_GROUPS tests above lift theirs. + */ + const decls = [...SOURCE.matchAll(/^(?:SWAP_SIZE|SWAP_FILE|SWAPPINESS)=.*$/gm)] + .map((m) => m[0]) + .join('\n'); + + const FNS = ['_size_mb', '_swap_sysctl', 'configure_swap', 'write_if_changed']; + + function stubs(dir: string): string { + return ` + CHANGED=(); log() { :; }; info() { echo "$*"; }; warn() { echo "$*"; } + note() { echo "changed: $*"; } + sysctl() { :; } + chown() { :; } + mkswap() { return \${MKSWAP_RC:-0}; } + swapon() { + case "\${1:-}" in + --show*) printf '%s' "\${FAKE_SWAPON_OUT:-}" ;; + *) return \${SWAPON_RC:-0} ;; + esac + } + systemd-detect-virt() { printf '%s' "\${FAKE_VIRT:-none}"; } + df() { if [[ -n "\${FAKE_DF:-}" ]]; then printf '%s\\n' "\$FAKE_DF"; else command df "\$@"; fi; } + eval "\$(declare -f configure_swap _swap_sysctl \\ + | sed 's#/etc/fstab#${dir}/fstab#g; s#/etc/sysctl.d/[a-z0-9-]*\\.conf#${dir}/sysctl.conf#g')" + ${decls} + `; + } + + /** A temp dir standing in for /etc, with an fstab that has no swap in it. */ + function box(): string { + const dir = mkdtempSync(join(tmpdir(), 'root-ubuntu-swap-')); + writeFileSync(join(dir, 'fstab'), '/dev/sda1 / ext4 defaults 0 1\n'); + return dir; + } + + const swap = (dir: string, env = '') => + shell(FNS, `${stubs(dir)}\n${env} SWAP_FILE=${dir}/swapfile configure_swap`); + + const fstabOf = (dir: string) => readFileSync(join(dir, 'fstab'), 'utf8'); + + it('makes a swapfile on a box that has none, and brings it back after a reboot', () => { + const dir = box(); + const out = swap(dir, 'SWAP_SIZE=8M'); + expect(out).toContain('changed: 8M swap at'); + expect(fstabOf(dir)).toMatch(/swapfile\s+none\s+swap\s+sw/); + expect(statSync(join(dir, 'swapfile')).size).toBe(8 * 1024 * 1024); + }); + + it('makes it unreadable, since swap is everything the box ever paged out', () => { + const dir = box(); + swap(dir, 'SWAP_SIZE=8M'); + expect(statSync(join(dir, 'swapfile')).mode & 0o777).toBe(0o600); + }); + + it('leaves a box that already has swap completely alone', () => { + // A swapfile stacked on a swap partition or a zram device is not more + // safety, it is a file nobody remembers making. + const dir = box(); + const out = swap(dir, "FAKE_SWAPON_OUT=$'/dev/sda2\\tpartition\\t4G\\n' SWAP_SIZE=8M"); + expect(out).toContain('swap already active: /dev/sda2 (partition, 4G)'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('still sets swappiness when the swap was already there', () => { + // The tuning is the point even when the swap is not ours: 60 pages out + // things that are still being used. + const dir = box(); + expect(swap(dir, "FAKE_SWAPON_OUT=$'/dev/sda2\\tpartition\\t4G\\n'")).toContain( + 'changed: vm.swappiness=10', + ); + expect(readFileSync(join(dir, 'sysctl.conf'), 'utf8')).toContain('vm.swappiness = 10'); + }); + + it('does nothing at all when SWAP_SIZE is 0', () => { + const dir = box(); + expect(swap(dir, 'SWAP_SIZE=0')).toContain('swap disabled'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('does not try to swap inside a container', () => { + // The kernel and its swap belong to the host. swapon in here either fails + // outright or is refused by the cgroup once the file already exists. + const dir = box(); + const out = swap(dir, 'FAKE_VIRT=lxc SWAP_SIZE=8M'); + expect(out).toContain('inside a lxc container'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('refuses btrfs and zfs, where a plain swapfile is wrong or dangerous', () => { + // btrfs needs chattr +C and no compression; a swapfile on zfs can deadlock + // the box under exactly the memory pressure it was added to survive. + const dir = box(); + expect(swap(dir, "FAKE_DF=$'FSTYPE\\nbtrfs' SWAP_SIZE=8M")).toContain('btrfs'); + expect(swap(dir, "FAKE_DF=$'FSTYPE\\nzfs' SWAP_SIZE=8M")).toContain('deadlock'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('will not fill the disk to buy memory headroom', () => { + // A full / breaks things a memory spike would never have touched. + const dir = box(); + expect(swap(dir, "FAKE_DF=$'ext4\\n900M' SWAP_SIZE=2G")).toContain('plus headroom -- skipping'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('fails loudly on a size it cannot read', () => { + const dir = box(); + expect(status(FNS, `${stubs(dir)}\nSWAP_SIZE=lots SWAP_FILE=${dir}/f configure_swap`)).toBe(1); + }); + + it('is safe to run twice: no second file, no second fstab line', () => { + // Everything else in this script converges on a re-run, and an appender + // that does not is how /etc/fstab grows a line a month. + const dir = box(); + swap(dir, 'SWAP_SIZE=8M'); + const out = swap(dir, 'SWAP_SIZE=8M'); + expect(out).toContain('fstab already brings'); + expect(fstabOf(dir).split('\n').filter((l) => l.includes('swapfile'))).toHaveLength(1); + }); + + it('cleans up the half-made file when mkswap fails', () => { + // A 2G file that is not swap is 2G of disk gone for nothing, and the next + // run would find it sitting there and take it for the real thing. + const dir = box(); + const out = shell( + FNS, + `${stubs(dir)} + MKSWAP_RC=1 SWAP_SIZE=8M SWAP_FILE=${dir}/swapfile configure_swap + [[ -e ${dir}/swapfile ]] && echo LEFTOVER || echo "cleaned up"`, + ); + expect(out).toContain('mkswap failed'); + expect(out).toContain('cleaned up'); + }); + + it('cleans up when swapon itself fails', () => { + const dir = box(); + const out = shell( + FNS, + `${stubs(dir)} + SWAPON_RC=1 SWAP_SIZE=8M SWAP_FILE=${dir}/swapfile configure_swap + [[ -e ${dir}/swapfile ]] && echo LEFTOVER || echo "cleaned up"`, + ); + expect(out).toContain('swapon failed'); + expect(out).toContain('cleaned up'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('refuses a path that is not a file instead of deleting it', () => { + // SWAP_FILE is configurable, and `rm -f` on a typo that happens to name a + // directory would be a very bad afternoon. + const dir = box(); + mkdirSync(join(dir, 'adirectory')); + const out = shell( + FNS, + `${stubs(dir)} + SWAP_SIZE=8M SWAP_FILE=${dir}/adirectory configure_swap + [[ -d ${dir}/adirectory ]] && echo "still a directory"`, + ); + expect(out).toContain('exists and is not a file'); + expect(out).toContain('still a directory'); + }); + + it('never leaves the swapfile readable, whatever the chown does', () => { + // Chained behind a chown, one failure there would leave the mode wide + // open -- and mkswap would still be perfectly happy with the file. + const body = SOURCE.slice(SOURCE.indexOf('configure_swap() {')); + const chmod = body.indexOf('chmod 0600 "$file"'); + const chown = body.indexOf('chown root:root "$file"'); + expect(chmod).toBeGreaterThan(-1); + expect(chown).toBeGreaterThan(chmod); + expect(body.slice(chmod, chown)).not.toContain('&&'); + }); + + it('matches the fstab entry on the path, so a hand-edited line survives', () => { + expect(SOURCE).toMatch(/awk -v f="\$file" '\$1 == f && \$3 == "swap"/); + }); + + it('is documented as a step and as an env override', () => { + const help = execFileSync('bash', [SCRIPT, '--help'], { encoding: 'utf8' }); + expect(help).toContain('swapfile'); + expect(help).toContain('SWAP_SIZE=2G'); + }); +});