diff --git a/root-ubuntu.sh b/root-ubuntu.sh index 8486672..ff15690 100755 --- a/root-ubuntu.sh +++ b/root-ubuntu.sh @@ -30,13 +30,14 @@ # automatically and refreshed # 2. apt update/upgrade + unattended security updates # 3. ufw -# 4. dotfiles (.zsh*, .bash*, .ssh*, ...) from $DOTFILES_REPO, if you have one -# 5. oh-my-zsh + plugins, oh-my-tmux, irssi configs -# 6. mise (curl https://mise.run | sh) -# 7. moshcode (curl https://moshcode.sh/install.sh | sh) -# 8. a per-user ssh-agent as a systemd user service -# 9. motd from $MOTD_URL -# 10. nginx per-user pages, per-user dev apps, TLS +# 4. a 2G swapfile, if the box has no swap at all +# 5. dotfiles (.zsh*, .bash*, .ssh*, ...) from $DOTFILES_REPO, if you have one +# 6. oh-my-zsh + plugins, oh-my-tmux, irssi configs +# 7. mise (curl https://mise.run | sh) +# 8. moshcode (curl https://moshcode.sh/install.sh | sh) +# 9. a per-user ssh-agent as a systemd user service +# 10. motd from $MOTD_URL +# 11. nginx per-user pages, per-user dev apps, TLS # # Usage, as root: # ./root-ubuntu.sh # first run, or a refresh @@ -94,6 +95,9 @@ # # Env overrides: # SSH_PORT=22 port to open in ufw +# SWAP_SIZE=2G swapfile to create when the box has no swap (0 = never) +# SWAP_FILE=/swapfile where that file goes +# SWAPPINESS=10 vm.swappiness once there is swap to speak of # ASSUME_YES=1 don't prompt (defaults: $DEFAULT_GROUPS; no privkey copy) # DEFAULT_GROUPS=... groups an account lands in when --groups is not passed # (default sudo,admin). An unattended run never prompts, so @@ -2131,6 +2135,141 @@ fi try "enable ufw at boot" systemctl enable ufw ufw status verbose || true +# ------------------------------------------------------------------ swap --- +# +# A box with no swap has no slack. The kernel's only answer to a memory spike +# is the OOM killer, and what it picks is whatever was biggest -- on these +# boxes, the build, the language server, or the editor someone was working in. +# 2G of swap does not make a small box a big one; it turns "the process died" +# into "that got slow for a moment", which is the difference between losing an +# afternoon and noticing nothing. +# +# Deliberately a swap FILE and not a partition: provider images arrive with the +# whole disk given to /, so there is no partition to make, and a file can be +# resized or removed on a live box. +SWAP_SIZE="${SWAP_SIZE:-2G}" # 0 disables; the box keeps whatever it has +SWAP_FILE="${SWAP_FILE:-/swapfile}" +# 60 (the default) treats swap as another tier of memory and pages out things +# that are still being used. 10 keeps it as the safety net it is meant to be. +SWAPPINESS="${SWAPPINESS:-10}" + +# 2G / 2048M / 2 (G assumed) -> megabytes +_size_mb() { + local s="${1^^}" + [[ "$s" =~ ^([0-9]+)([GM]?)$ ]] || return 1 + case "${BASH_REMATCH[2]}" in + M) printf '%s' "${BASH_REMATCH[1]}" ;; + *) printf '%s' "$(( BASH_REMATCH[1] * 1024 ))" ;; + esac +} + +_swap_sysctl() { + write_if_changed /etc/sysctl.d/60-profullstack-swap.conf </dev/null || true + note "vm.swappiness=$SWAPPINESS" +} + +configure_swap() { + local file="$SWAP_FILE" dir want_mb avail_mb fstype virt name type size _rest + + if [[ -z "$SWAP_SIZE" || "$SWAP_SIZE" == 0 ]]; then + info "swap disabled (SWAP_SIZE=$SWAP_SIZE) -- leaving this box as it is" + return 0 + fi + + # Somebody else's swap counts. A swapfile stacked on top of a swap + # partition or a zram device is not extra safety, it is a file nobody + # remembers making. + if swapon --show=NAME --noheadings 2>/dev/null | grep -q .; then + while read -r name type size _rest; do + info "swap already active: $name ($type, $size)" + done < <(swapon --show=NAME,TYPE,SIZE --noheadings 2>/dev/null) + _swap_sysctl + return 0 + fi + + # Containers share the host kernel, and its swap is the host's business. + # swapon in here either fails outright or is refused by the cgroup after + # the file has already been written. + virt="$(systemd-detect-virt --container 2>/dev/null)" + if [[ -n "$virt" && "$virt" != none ]]; then + info "inside a $virt container -- swap belongs to the host" + return 0 + fi + + dir="$(dirname "$file")" + fstype="$(df --output=fstype "$dir" 2>/dev/null | tail -1)" + case "$fstype" in + btrfs) + warn "no swapfile: btrfs needs one built its own way (chattr +C, no compression, no snapshots)" + return 0 ;; + zfs) + warn "no swapfile: a swapfile on zfs can deadlock the box -- use a zvol" + return 0 ;; + esac + + want_mb="$(_size_mb "$SWAP_SIZE")" || { + warn "swap: cannot read SWAP_SIZE=$SWAP_SIZE (want something like 2G or 2048M)" + return 1 + } + + # Filling the root disk to buy memory headroom is a bad trade: a full / + # breaks things a memory spike would not have touched. + avail_mb="$(df -BM --output=avail "$dir" 2>/dev/null | tail -1 | tr -dc '0-9')" + if [[ -n "$avail_mb" ]] && (( avail_mb < want_mb + 2048 )); then + warn "swap: ${avail_mb}M free on $dir, need ${want_mb}M plus headroom -- skipping" + return 0 + fi + + [[ -e "$file" && ! -f "$file" ]] && { warn "swap: $file exists and is not a file"; return 1; } + + log "creating ${SWAP_SIZE} of swap at $file" + # fallocate is instant, but on some filesystems it leaves unwritten extents + # that mkswap then refuses. dd always works and is only slow the once, so + # it is both the fallback and the retry. + rm -f "$file" + fallocate -l "${want_mb}M" "$file" 2>/dev/null \ + || dd if=/dev/zero of="$file" bs=1M count="$want_mb" status=none \ + || { warn "swap: could not allocate $file"; rm -f "$file"; return 1; } + # World-readable swap is every secret the machine has ever paged out, so + # the mode goes on first and on its own -- chained behind a chown, one + # failure there would leave the file readable and mkswap would still be + # happy with it. + chmod 0600 "$file" + chown root:root "$file" + if ! mkswap "$file" >/dev/null 2>&1; then + # the unwritten-extents case: write the bytes for real, then try once more + dd if=/dev/zero of="$file" bs=1M count="$want_mb" status=none + chmod 0600 "$file" + if ! mkswap "$file" >/dev/null 2>&1; then + warn "swap: mkswap failed on $file" + rm -f "$file" + return 1 + fi + fi + swapon "$file" || { warn "swap: swapon failed on $file"; rm -f "$file"; return 1; } + note "${SWAP_SIZE} swap at $file" + + # ...and again after a reboot. Matched on the path, so an entry someone has + # since edited (different options, a different priority) is left alone. + if awk -v f="$file" '$1 == f && $3 == "swap" { found = 1 } END { exit !found }' /etc/fstab; then + info "fstab already brings $file up at boot" + else + printf '%-16s none swap sw 0 0\n' "$file" >>/etc/fstab \ + && note "fstab: $file" + fi + + _swap_sysctl + return 0 +} + +log "configuring swap" +try "swap" configure_swap + # ------------------------------------------------------------------ motd --- # Fetched into a cache file; login just cats it and kicks off a background @@ -4494,6 +4633,8 @@ info "open: $(ufw status | awk '/ALLOW/{printf "%s ", $1}')" info "mosh: $(command -v mosh-server >/dev/null && echo "$(mosh-server --version 2>&1 | head -1)" || echo 'MISSING')" info "tail: $(tailscale ip -4 2>/dev/null | head -1 || echo 'not joined to a tailnet')" info "motd: $([[ -s $MOTD_CACHE ]] && echo "cached ($(wc -l <"$MOTD_CACHE") lines)" || echo 'empty')" +info "ram: $(free -h | awk '/^Mem:/{printf "%s total, %s available", $2, $7}')" +info "swap: $(swapon --show=NAME,SIZE --noheadings 2>/dev/null | awk '{printf "%s (%s) ", $1, $2}' | grep . || echo 'NONE -- one memory spike from the OOM killer')" echo printf ' %-12s %-6s %-6s %-9s %-6s %-5s %s\n' USER OMZ MISE MOSHCODE TMUX APPS SHELL while read -r login; do diff --git a/server.conf.example b/server.conf.example index ee988b6..383d4d8 100644 --- a/server.conf.example +++ b/server.conf.example @@ -32,6 +32,17 @@ # will happily lock you out of the box you are provisioning. #SSH_PORT=22 +# A box with no swap has no slack: the kernel's only answer to a memory spike +# is the OOM killer, and what it picks is whatever was biggest. A swapfile is +# created only when the box has no swap at all, so a machine that already has +# a swap partition or zram is left exactly as it is. Set 0 to never make one. +#SWAP_SIZE=2G +#SWAP_FILE=/swapfile + +# 60, the kernel default, treats swap as another tier of memory and pages out +# things that are still being used. 10 keeps it as the safety net it is for. +#SWAPPINESS=10 + # ── accounts ───────────────────────────────────────────────────────────────── # Groups new accounts land in when nothing is passed on the command line. diff --git a/test/root-ubuntu.test.ts b/test/root-ubuntu.test.ts index f4eb5e7..86cfed7 100644 --- a/test/root-ubuntu.test.ts +++ b/test/root-ubuntu.test.ts @@ -1,6 +1,7 @@ import { execFileSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; +import { mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; @@ -577,3 +578,218 @@ describe('the groups subcommand', () => { expect(out).toContain('groups add alice docker'); }); }); + +describe('_size_mb', () => { + const mb = (s: string) => shell(['_size_mb'], `_size_mb ${s}`); + + it('reads the sizes a person would actually write', () => { + expect(mb('2G')).toBe('2048'); + expect(mb('2g')).toBe('2048'); + expect(mb('512M')).toBe('512'); + }); + + it('assumes gigabytes for a bare number, because nobody means 2MB of swap', () => { + expect(mb('4')).toBe('4096'); + }); + + it('refuses anything it cannot read rather than guessing', () => { + // The value ends up in `dd count=`, so a silent misread is a swapfile of + // the wrong size -- or, with an empty count, no swapfile at all. + expect(status(['_size_mb'], '_size_mb abc')).toBe(1); + expect(status(['_size_mb'], '_size_mb "2 G"')).toBe(1); + expect(status(['_size_mb'], '_size_mb ""')).toBe(1); + expect(status(['_size_mb'], '_size_mb 2GB')).toBe(1); + }); +}); + +describe('configure_swap', () => { + /** + * The function reaches for swapon, mkswap, df and systemd-detect-virt, and + * writes to /etc/fstab and /etc/sysctl.d. Those tools are stubbed and the + * two absolute paths are rewritten into a temp directory with `declare -f`, + * so what runs is the code in the file and the test still cannot touch the + * machine it runs on. The SWAP_* declarations are top-level rather than + * inside a function, so they are lifted out of the source the same way the + * DEFAULT_GROUPS tests above lift theirs. + */ + const decls = [...SOURCE.matchAll(/^(?:SWAP_SIZE|SWAP_FILE|SWAPPINESS)=.*$/gm)] + .map((m) => m[0]) + .join('\n'); + + const FNS = ['_size_mb', '_swap_sysctl', 'configure_swap', 'write_if_changed']; + + function stubs(dir: string): string { + return ` + CHANGED=(); log() { :; }; info() { echo "$*"; }; warn() { echo "$*"; } + note() { echo "changed: $*"; } + sysctl() { :; } + chown() { :; } + mkswap() { return \${MKSWAP_RC:-0}; } + swapon() { + case "\${1:-}" in + --show*) printf '%s' "\${FAKE_SWAPON_OUT:-}" ;; + *) return \${SWAPON_RC:-0} ;; + esac + } + systemd-detect-virt() { printf '%s' "\${FAKE_VIRT:-none}"; } + df() { if [[ -n "\${FAKE_DF:-}" ]]; then printf '%s\\n' "\$FAKE_DF"; else command df "\$@"; fi; } + eval "\$(declare -f configure_swap _swap_sysctl \\ + | sed 's#/etc/fstab#${dir}/fstab#g; s#/etc/sysctl.d/[a-z0-9-]*\\.conf#${dir}/sysctl.conf#g')" + ${decls} + `; + } + + /** A temp dir standing in for /etc, with an fstab that has no swap in it. */ + function box(): string { + const dir = mkdtempSync(join(tmpdir(), 'root-ubuntu-swap-')); + writeFileSync(join(dir, 'fstab'), '/dev/sda1 / ext4 defaults 0 1\n'); + return dir; + } + + const swap = (dir: string, env = '') => + shell(FNS, `${stubs(dir)}\n${env} SWAP_FILE=${dir}/swapfile configure_swap`); + + const fstabOf = (dir: string) => readFileSync(join(dir, 'fstab'), 'utf8'); + + it('makes a swapfile on a box that has none, and brings it back after a reboot', () => { + const dir = box(); + const out = swap(dir, 'SWAP_SIZE=8M'); + expect(out).toContain('changed: 8M swap at'); + expect(fstabOf(dir)).toMatch(/swapfile\s+none\s+swap\s+sw/); + expect(statSync(join(dir, 'swapfile')).size).toBe(8 * 1024 * 1024); + }); + + it('makes it unreadable, since swap is everything the box ever paged out', () => { + const dir = box(); + swap(dir, 'SWAP_SIZE=8M'); + expect(statSync(join(dir, 'swapfile')).mode & 0o777).toBe(0o600); + }); + + it('leaves a box that already has swap completely alone', () => { + // A swapfile stacked on a swap partition or a zram device is not more + // safety, it is a file nobody remembers making. + const dir = box(); + const out = swap(dir, "FAKE_SWAPON_OUT=$'/dev/sda2\\tpartition\\t4G\\n' SWAP_SIZE=8M"); + expect(out).toContain('swap already active: /dev/sda2 (partition, 4G)'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('still sets swappiness when the swap was already there', () => { + // The tuning is the point even when the swap is not ours: 60 pages out + // things that are still being used. + const dir = box(); + expect(swap(dir, "FAKE_SWAPON_OUT=$'/dev/sda2\\tpartition\\t4G\\n'")).toContain( + 'changed: vm.swappiness=10', + ); + expect(readFileSync(join(dir, 'sysctl.conf'), 'utf8')).toContain('vm.swappiness = 10'); + }); + + it('does nothing at all when SWAP_SIZE is 0', () => { + const dir = box(); + expect(swap(dir, 'SWAP_SIZE=0')).toContain('swap disabled'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('does not try to swap inside a container', () => { + // The kernel and its swap belong to the host. swapon in here either fails + // outright or is refused by the cgroup once the file already exists. + const dir = box(); + const out = swap(dir, 'FAKE_VIRT=lxc SWAP_SIZE=8M'); + expect(out).toContain('inside a lxc container'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('refuses btrfs and zfs, where a plain swapfile is wrong or dangerous', () => { + // btrfs needs chattr +C and no compression; a swapfile on zfs can deadlock + // the box under exactly the memory pressure it was added to survive. + const dir = box(); + expect(swap(dir, "FAKE_DF=$'FSTYPE\\nbtrfs' SWAP_SIZE=8M")).toContain('btrfs'); + expect(swap(dir, "FAKE_DF=$'FSTYPE\\nzfs' SWAP_SIZE=8M")).toContain('deadlock'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('will not fill the disk to buy memory headroom', () => { + // A full / breaks things a memory spike would never have touched. + const dir = box(); + expect(swap(dir, "FAKE_DF=$'ext4\\n900M' SWAP_SIZE=2G")).toContain('plus headroom -- skipping'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('fails loudly on a size it cannot read', () => { + const dir = box(); + expect(status(FNS, `${stubs(dir)}\nSWAP_SIZE=lots SWAP_FILE=${dir}/f configure_swap`)).toBe(1); + }); + + it('is safe to run twice: no second file, no second fstab line', () => { + // Everything else in this script converges on a re-run, and an appender + // that does not is how /etc/fstab grows a line a month. + const dir = box(); + swap(dir, 'SWAP_SIZE=8M'); + const out = swap(dir, 'SWAP_SIZE=8M'); + expect(out).toContain('fstab already brings'); + expect(fstabOf(dir).split('\n').filter((l) => l.includes('swapfile'))).toHaveLength(1); + }); + + it('cleans up the half-made file when mkswap fails', () => { + // A 2G file that is not swap is 2G of disk gone for nothing, and the next + // run would find it sitting there and take it for the real thing. + const dir = box(); + const out = shell( + FNS, + `${stubs(dir)} + MKSWAP_RC=1 SWAP_SIZE=8M SWAP_FILE=${dir}/swapfile configure_swap + [[ -e ${dir}/swapfile ]] && echo LEFTOVER || echo "cleaned up"`, + ); + expect(out).toContain('mkswap failed'); + expect(out).toContain('cleaned up'); + }); + + it('cleans up when swapon itself fails', () => { + const dir = box(); + const out = shell( + FNS, + `${stubs(dir)} + SWAPON_RC=1 SWAP_SIZE=8M SWAP_FILE=${dir}/swapfile configure_swap + [[ -e ${dir}/swapfile ]] && echo LEFTOVER || echo "cleaned up"`, + ); + expect(out).toContain('swapon failed'); + expect(out).toContain('cleaned up'); + expect(fstabOf(dir)).not.toContain('swapfile'); + }); + + it('refuses a path that is not a file instead of deleting it', () => { + // SWAP_FILE is configurable, and `rm -f` on a typo that happens to name a + // directory would be a very bad afternoon. + const dir = box(); + mkdirSync(join(dir, 'adirectory')); + const out = shell( + FNS, + `${stubs(dir)} + SWAP_SIZE=8M SWAP_FILE=${dir}/adirectory configure_swap + [[ -d ${dir}/adirectory ]] && echo "still a directory"`, + ); + expect(out).toContain('exists and is not a file'); + expect(out).toContain('still a directory'); + }); + + it('never leaves the swapfile readable, whatever the chown does', () => { + // Chained behind a chown, one failure there would leave the mode wide + // open -- and mkswap would still be perfectly happy with the file. + const body = SOURCE.slice(SOURCE.indexOf('configure_swap() {')); + const chmod = body.indexOf('chmod 0600 "$file"'); + const chown = body.indexOf('chown root:root "$file"'); + expect(chmod).toBeGreaterThan(-1); + expect(chown).toBeGreaterThan(chmod); + expect(body.slice(chmod, chown)).not.toContain('&&'); + }); + + it('matches the fstab entry on the path, so a hand-edited line survives', () => { + expect(SOURCE).toMatch(/awk -v f="\$file" '\$1 == f && \$3 == "swap"/); + }); + + it('is documented as a step and as an env override', () => { + const help = execFileSync('bash', [SCRIPT, '--help'], { encoding: 'utf8' }); + expect(help).toContain('swapfile'); + expect(help).toContain('SWAP_SIZE=2G'); + }); +});