dnsmasq in the net zone drops to nobody once its sockets are bound, and reads its servers' file itself when it changes

build/guest-tests/zones-check.sh

@@ -53,6 +53,51 @@ done
 if [[ "$ready" == *"nat=yes"* ]]; then pass "net-ready" "$ready"; else fail "net-ready" "no READY line with nat=yes in the catch-all log (last: $(netzone_said '' | tail -1))"; fi
 # The routed zones' resolver, named on its own: routed-dns only times out.
 if [[ "$ready" == *" dns=yes "* ]]; then pass "net-dns" "dnsmasq is running"; else fail "net-dns" "$(uncaught | grep -a 'dnsmasq' | tail -2 | tr '\n' ' ')"; fi
+# dhcpcd's privilege separation: what parses a lease runs as the net zone's
+# dhcpcd user (host uid_base + 100) in an empty root, with no capability and
+# dhcpcd's own seccomp filter over the zone's, while a helper stays its root;
+# and the lease, which that helper writes, arrived.
+net_base="$(sed -n 's/^uid_base *= *\([0-9]*\).*/\1/p' "$Z/net.toml")"
+nz_init="$(cut -d' ' -f1 /run/kryptik/zones/net/init.pid 2>/dev/null)"
+separated=0; helpers=0; seen=""
+for p in $(pgrep -x dhcpcd); do
+    uid="$(awk '/^Uid:/ { print $2 }' "/proc/$p/status" 2>/dev/null)"
+    caps="$(awk '/^CapEff:/ { print $2 }' "/proc/$p/status" 2>/dev/null)"
+    filters="$(awk '/^Seccomp_filters:/ { print $2 }' "/proc/$p/status" 2>/dev/null)"
+    # 2>&1: a root that cannot be listed is not an empty one.
+    inside="$(ls -A "/proc/$p/root" 2>&1 | head -3 | tr '\n' ',')"
+    seen="${seen} ${p}:uid=${uid},caps=${caps},filters=${filters},root=$(readlink "/proc/$p/root" 2>/dev/null)[${inside}]"
+    if [[ "$uid" == "$((net_base + 100))" && "$caps" == 0000000000000000 && "${filters:-0}" -ge 2 && -z "$inside" ]]; then
+        separated=$((separated + 1))
+    fi
+    [[ "$uid" == "$net_base" ]] && helpers=$((helpers + 1))
+done
+leased="$(nsenter -t "${nz_init:-0}" -m sh -c 'ls /var/lib/dhcpcd/*.lease 2>/dev/null' | head -1)"
+if [[ "$separated" -ge 1 && "$helpers" -ge 1 && -n "$leased" ]]; then
+    pass "dhcpcd-separated" "${separated} dhcpcd process(es) as uid $((net_base + 100)) in an empty root with no capability under two filters, ${helpers} root helper, lease ${leased}"
+else
+    fail "dhcpcd-separated" "separated ${separated}, helpers ${helpers}, lease ${leased:-none}:${seen:- no dhcpcd running}"
+fi
+# dnsmasq answers the routed zones as the net zone's nobody (host uid_base +
+# 65534), keeping at most CAP_NET_BIND_SERVICE: fd19::1 stays tentative on a
+# bridge with no port yet, so dnsmasq binds it later.
+dns_dropped=0; dns_root=0; dns_seen=""
+for p in $(pgrep -x dnsmasq); do
+    ids="$(awk '/^Uid:/ { print $2, $3, $4, $5 }' "/proc/$p/status" 2>/dev/null)"
+    eff="$(awk '/^CapEff:/ { print $2 }' "/proc/$p/status" 2>/dev/null)"
+    prm="$(awk '/^CapPrm:/ { print $2 }' "/proc/$p/status" 2>/dev/null)"
+    dns_seen="${dns_seen} ${p}:uid=${ids// /,},eff=${eff},prm=${prm}"
+    n=$((net_base + 65534))
+    if [[ "$ids" == "$n $n $n $n" && "$eff" =~ ^0000000000000(000|400)$ && "$prm" =~ ^0000000000000(000|400)$ ]]; then
+        dns_dropped=$((dns_dropped + 1))
+    fi
+    [[ "$ids" == "$net_base "* ]] && dns_root=$((dns_root + 1))
+done
+if [[ "$dns_dropped" -ge 1 && "$dns_root" -eq 0 ]]; then
+    pass "dnsmasq-unprivileged" "${dns_dropped} dnsmasq process(es) as uid $((net_base + 65534)) with no capability but CAP_NET_BIND_SERVICE, none as the net zone's root"
+else
+    fail "dnsmasq-unprivileged" "dropped ${dns_dropped}, as root ${dns_root}:${dns_seen:- no dnsmasq running}"
+fi
 if ip link show eth0 >/dev/null 2>&1; then fail "zone0-nic" "eth0 is still in zone 0"; else pass "zone0-nic" "eth0 is not in zone 0 (moved into the net zone)"; fi
 if [[ -z "$(ip route show default 2>/dev/null)" ]]; then pass "zone0-no-route" "zone 0 has no default route"; else fail "zone0-no-route" "$(ip route show default)"; fi
 if ping -c1 -W2 10.0.2.2 >/dev/null 2>&1; then fail "zone0-offline" "zone 0 reached the VM gateway"; else pass "zone0-offline" "zone 0 cannot reach the VM gateway"; fi
@@ -96,8 +141,9 @@ fi
 # The resolver follows the servers a lease names. The net zone's resolv.conf
 # is written with the servers dnsmasq has and one more, as a lease that came
 # late or another network would change it, and then without it: each time
-# dnsmasq must be told within a few of the zone's 10 s passes. Its own servers
-# stay throughout, so names still resolve.
+# the zone's 10 s pass writes dnsmasq's file, and dnsmasq, woken by a query,
+# reads it and logs the servers it now uses. Its own servers stay throughout,
+# so names still resolve.
 forwards_to() {   # forwards_to yes|no: wait until dnsmasq's file does, or does not, name the added server
     for _ in $(seq 1 40); do
         if netsh 'grep -q "^nameserver 192\.0\.2\.53$" /run/uplink-resolv.conf' > /dev/null; then [[ "$1" == yes ]] && return 0
@@ -106,18 +152,36 @@ forwards_to() {   # forwards_to yes|no: wait until dnsmasq's file does, or does
     done
     return 1
 }
-told() { grep -hc 'netzone: dnsmasq: now forwarding to' /run/uncaught-logs/current /run/uncaught-logs/@* 2>/dev/null | awk '{ n += $1 } END { print n + 0 }'; }
-told_before="$(told)"
+reads() { uncaught | grep -ac 'dnsmasq\[[0-9]*\]: reading /run/uplink-resolv\.conf'; }
+# dnsmasq's lines from its last read of the file on.
+last_read() { uncaught | grep -a 'dnsmasq\[[0-9]*\]: ' | awk '/: reading \/run\/uplink-resolv\.conf/ { s = "" } { s = s $0 "\n" } END { printf "%s", s }'; }
+poke='import socket, struct
+s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.settimeout(2)
+s.sendto(struct.pack(">HHHHHH", 7, 0x100, 1, 0, 0, 0) + b"\x07kryptik\x04test\x00\x00\x01\x00\x01", ("127.0.0.1", 53))
+s.recv(512)'
+read_after() {   # read_after N: query dnsmasq, which looks at its file at most once a second, until it has read it more than N times
+    for _ in $(seq 1 20); do
+        nsenter -t "${net_init:-0}" -n python3 -c "$poke" > /dev/null 2>&1
+        [[ "$(reads)" -gt "$1" ]] && { sleep 2; return 0; }   # its server lines follow
+        sleep 1
+    done
+    return 1
+}
 was="$(netsh 'grep "^nameserver" /run/uplink-resolv.conf')"
+before="$(reads)"
 if [[ "$was" == nameserver* ]] && wrote="$(netsh "printf '%s\n' '${was}' 'nameserver 192.0.2.53' > /etc/resolv.conf")"; then
     forwards_to yes; came=$?
+    read_after "$before"; read1=$?
+    gained="$(last_read | grep -c 'using nameserver 192\.0\.2\.53#53')"
+    before="$(reads)"
     netsh "printf '%s\n' '${was}' > /etc/resolv.conf" > /dev/null
     forwards_to no; went=$?
-    sleep 2   # the zone says so after it has told dnsmasq
-    if [[ "$came" -eq 0 && "$went" -eq 0 && "$(( $(told) - told_before ))" -ge 2 ]]; then
-        pass "dns-follows-lease" "a server the net zone's resolv.conf gained reached dnsmasq, and left it again with the lease"
+    read_after "$before"; read2=$?
+    kept="$(last_read | grep -c 'using nameserver 192\.0\.2\.53#53')"
+    if [[ "$came" -eq 0 && "$went" -eq 0 && "$read1" -eq 0 && "$read2" -eq 0 && "$gained" -ge 1 && "$kept" -eq 0 ]]; then
+        pass "dns-follows-lease" "a server the net zone's resolv.conf gained reached dnsmasq, which used it, and left it again with the lease"
     else
-        fail "dns-follows-lease" "gained: rc=$came, lost: rc=$went, told $(( $(told) - told_before )) time(s); $(grep -h 'netzone: dnsmasq' /run/uncaught-logs/current 2>/dev/null | tail -2 | tr '\n' ' ')"
+        fail "dns-follows-lease" "file gained: rc=$came, read: rc=$read1, used: $gained; file lost: rc=$went, read: rc=$read2, still used: $kept; $(last_read | tail -3 | tr '\n' ' ')"
     fi
 else
     fail "dns-follows-lease" "dnsmasq's servers could not be read in the net zone (init ${net_init:-none}), or its resolv.conf not written: ${was:-nothing read} ${wrote:-}"
@@ -718,7 +782,7 @@ zrun untrusted 30 -- grep -c /usr/lib/libhardened_malloc.so /proc/self/maps
 [[ "$ZRC" = 0 ]] && pass "allocator-zone" "a process in untrusted runs on it too" || fail "allocator-zone" "rc=$ZRC $(tail -1 "$LOG/untrusted.err")"
 
 # --- the installed root: privilege only where the allowlist says -------------------
-# What stage 06 stripped stays stripped: on the root filesystem a setuid or
+# Stage 06 fails the build on any other bit; the installed root shows it: a setuid or
 # setgid bit is on the listed binaries alone (build/config/setuid-allowlist.txt)
 # and file capabilities are on none (capability-allowlist.txt is empty).
 setuid_found="$(find / -xdev -type f -perm /6000 2>/dev/null | LC_ALL=C sort | tr '\n' ' ')"

build/recipes/netzone.sh

@@ -12,6 +12,9 @@ s_netzone() {
     python3 -m py_compile /usr/libexec/kryptik/sntp-offset.py || { echo "sntp-offset.py does not compile under the target python"; return 1; }
     install -D -m 0755 "${KRYPTIK_ROOT}/tools/net/update-fetch.py" /usr/libexec/kryptik/update-fetch.py
     python3 -m py_compile /usr/libexec/kryptik/update-fetch.py || { echo "update-fetch.py does not compile under the target python"; return 1; }
+    # dhcpcd's hook: its root helper runs it with what the unprivileged side sends.
+    install -D -m 0755 "${KRYPTIK_ROOT}/tools/net/dhcpcd-hook.py" /usr/libexec/kryptik/dhcpcd-hook
+    python3 -m py_compile /usr/libexec/kryptik/dhcpcd-hook || { echo "dhcpcd-hook does not compile under the target python"; return 1; }
     rm -rf /usr/libexec/kryptik/__pycache__
     for t in dhcpcd nft dnsmasq ip; do
         command -v "$t" >/dev/null 2>&1 && echo "  ok $t" || { echo "  MISSING $t"; return 1; }

compartments/kryptikd/src/caps.rs

@@ -5,7 +5,7 @@ use std::io;
 
 /// Capability numbers from `linux/capability.h` (stable ABI; `libc` lacks them).
 #[allow(dead_code)]
-mod cap {
+pub(crate) mod cap {
     use libc::c_int;
     pub const CHOWN: c_int = 0;
     pub const DAC_OVERRIDE: c_int = 1;
@@ -21,6 +21,7 @@ mod cap {
     pub const NET_RAW: c_int = 13;
     pub const IPC_LOCK: c_int = 14;
     pub const SYS_MODULE: c_int = 16;
+    pub const SYS_CHROOT: c_int = 18;
     pub const SYS_PTRACE: c_int = 19;
     pub const SYS_ADMIN: c_int = 21;
     pub const SYS_NICE: c_int = 23;
@@ -35,9 +36,17 @@ pub const KEEP: libc::c_int = cap::NET_BIND_SERVICE;
 pub const KEEPABLE: &[libc::c_int] = &[
     cap::NET_BIND_SERVICE, cap::NET_ADMIN, cap::NET_RAW, cap::NET_BROADCAST,
     cap::SYS_NICE, cap::IPC_LOCK, cap::KILL, cap::CHOWN, cap::FOWNER, cap::FSETID,
-    cap::DAC_READ_SEARCH,
+    cap::DAC_READ_SEARCH, cap::SETUID, cap::SETGID, cap::SYS_CHROOT,
 ];
 
+/// What a daemon needs to drop to a user of its own (dhcpcd's privilege separation): kept
+/// together or not at all, and each opens the calls `seccomp::CAP_CALLS` names.
+pub const PRIVSEP: &[libc::c_int] = &[cap::SETUID, cap::SETGID, cap::SYS_CHROOT];
+
+pub fn keeps_privsep(keep: &[libc::c_int]) -> bool {
+    PRIVSEP.iter().all(|c| keep.contains(c))
+}
+
 /// Capability numbers by name, as linux/capability.h defines them.
 pub const CAP_NAMES: &[(&str, libc::c_int)] = &[
     ("CAP_CHOWN", 0), ("CAP_DAC_OVERRIDE", 1), ("CAP_DAC_READ_SEARCH", 2), ("CAP_FOWNER", 3),
@@ -53,9 +62,9 @@ pub const CAP_NAMES: &[(&str, libc::c_int)] = &[
     ("CAP_CHECKPOINT_RESTORE", 40),
 ];
 
-/// Only the nic zone may keep these (`policy::check_for_zone`): another zone could use them to
-/// re-address its veth or forge frames.
-pub const NIC_ONLY: &[libc::c_int] = &[cap::NET_ADMIN, cap::NET_RAW];
+/// Only the nic zone may keep these (`policy::check_for_zone`): another zone could use the
+/// network ones to re-address its veth or forge frames, and only dhcpcd there needs `PRIVSEP`.
+pub const NIC_ONLY: &[libc::c_int] = &[cap::NET_ADMIN, cap::NET_RAW, cap::SETUID, cap::SETGID, cap::SYS_CHROOT];
 
 pub fn cap_by_name(name: &str) -> Option<libc::c_int> {
     CAP_NAMES.iter().find(|(n, _)| *n == name).map(|(_, v)| *v)

compartments/kryptikd/src/caps/tests.rs

@@ -23,6 +23,21 @@ fn keep_is_not_dangerous() {
     }
 }
 
+#[test]
+fn privsep_is_the_nic_zones_alone() {
+    assert_eq!(PRIVSEP, &[cap::SETUID, cap::SETGID, cap::SYS_CHROOT]);
+    for c in PRIVSEP {
+        assert!(KEEPABLE.contains(c) && NIC_ONLY.contains(c), "{}", cap_name(*c));
+    }
+    assert_eq!(cap_by_name("CAP_SYS_CHROOT"), Some(cap::SYS_CHROOT));
+    assert!(keeps_privsep(&[cap::NET_ADMIN, cap::SYS_CHROOT, cap::SETGID, cap::SETUID]));
+    assert!(!keeps_privsep(&[cap::SETUID, cap::SETGID]));
+    // Nothing else comes with them: the dangerous ones stay unkeepable.
+    for c in [cap::SYS_ADMIN, cap::SYS_PTRACE, cap::DAC_OVERRIDE, cap::SYS_MODULE, cap::MKNOD] {
+        assert!(!KEEPABLE.contains(&c), "{}", cap_name(c));
+    }
+}
+
 #[test]
 fn last_cap_is_sane() {
     let n = last_cap();

compartments/kryptikd/src/isolate.rs

@@ -72,22 +72,33 @@ pub fn unshare_namespaces(flags: libc::c_int) -> Result<(), IsolateError> {
     check("unshare", unsafe { libc::unshare(flags) })
 }
 
+/// The id a daemon in a zone that keeps `caps::PRIVSEP` drops to: dhcpcd's, in the nic zone.
+pub const SERVICE_ID: u32 = 100;
+
 /// Write a new user namespace's id maps, denying setgroups first as an unprivileged gid_map
-/// needs. `with_nobody` maps 65534 too, which needs CAP_SETUID: a root launch only.
+/// needs. `with_nobody` maps 65534 too, which needs CAP_SETUID: a root launch only. `service`
+/// maps `SERVICE_ID` as well and leaves setgroups allowed, for that daemon's drop: a root launch
+/// of a zone that keeps `caps::PRIVSEP` only.
 pub fn write_id_maps(
     pid: libc::pid_t,
     outer_uid: u32,
     outer_gid: u32,
     with_nobody: bool,
+    service: bool,
 ) -> Result<(), IsolateError> {
     use std::fs;
 
-    let deny = format!("/proc/{pid}/setgroups");
-    fs::write(&deny, "deny")
-        .map_err(|e| IsolateError::Refused(format!("{deny}: {e}")))?;
+    if !service {
+        let deny = format!("/proc/{pid}/setgroups");
+        fs::write(&deny, "deny")
+            .map_err(|e| IsolateError::Refused(format!("{deny}: {e}")))?;
+    }
 
     let map = |outer: u32| {
         let mut m = format!("0 {outer} 1\n");
+        if service {
+            m.push_str(&format!("{SERVICE_ID} {} 1\n", outer + SERVICE_ID));
+        }
         if with_nobody {
             m.push_str(&format!("65534 {} 1\n", outer + 65534));
         }

compartments/kryptikd/src/main.rs

@@ -1148,7 +1148,7 @@ const fn seccomp_iowr(nr: u32, size: usize) -> libc::c_ulong {
 const NOTIF_RECV: libc::c_ulong = seccomp_iowr(0, std::mem::size_of::<libc::seccomp_notif>());
 const NOTIF_SEND: libc::c_ulong = seccomp_iowr(1, std::mem::size_of::<libc::seccomp_notif_resp>());
 
-/// The filter zone `name` runs under: the base, widened by its policy file.
+/// The filter zone `name` runs under: the base, widened by its policy file and kept capabilities.
 fn trace_filter(dir: &Path, name: &str) -> Result<(Vec<libc::c_long>, seccomp::SocketPolicy), String> {
     let zones = zone::load_all(dir).map_err(|e| e.to_string())?;
     let z = zones.iter().find(|z| z.name == name).ok_or_else(|| format!("no zone named {name:?}"))?;
@@ -1158,7 +1158,7 @@ fn trace_filter(dir: &Path, name: &str) -> Result<(Vec<libc::c_long>, seccomp::S
     let p = policy::load(&policy::resolve(dir, rel))
         .and_then(|p| p.check_for_zone(z).map(|_| p))
         .map_err(|e| format!("{rel}: {e}"))?;
-    Ok((seccomp::widened(&p.extra_syscalls).map_err(|e| e.to_string())?, p.sockets))
+    Ok((seccomp::widened_for(&p.extra_syscalls, &p.keep_caps).map_err(|e| e.to_string())?, p.sockets))
 }
 
 /// Run CMD under a zone filter, naming every call it refuses. Refusals arrive by user

compartments/kryptikd/src/policy.rs

@@ -154,12 +154,25 @@ pub fn parse(text: &str, source: &str) -> Result<Policy, PolicyError> {
             }
         }
     }
+    // One of them alone opens its calls and serves no daemon's drop to its own user.
+    let privsep = caps::PRIVSEP.iter().filter(|c| p.keep_caps.contains(c)).count();
+    if privsep != 0 && privsep != caps::PRIVSEP.len() {
+        return Err(PolicyError::Line {
+            path: source.to_string(),
+            line: 0,
+            msg: format!(
+                "{} are kept together or not at all",
+                caps::PRIVSEP.iter().map(|c| caps::cap_name(*c)).collect::<Vec<_>>().join(", ")
+            ),
+        });
+    }
     Ok(p)
 }
 
 impl Policy {
-    /// Only the nic zone may keep `CAP_NET_ADMIN` or `CAP_NET_RAW`; elsewhere they let a zone
-    /// re-address its veth, route around port isolation via the bridge address, or forge frames.
+    /// Only the nic zone may keep `caps::NIC_ONLY`: elsewhere `CAP_NET_ADMIN` or `CAP_NET_RAW` let a
+    /// zone re-address its veth, route around port isolation via the bridge address, or forge
+    /// frames, and no other zone runs a daemon that drops to a user of its own.
     pub fn check_for_zone(&self, zone: &crate::zone::Zone) -> Result<(), PolicyError> {
         if zone.network != crate::zone::NetworkMode::Nic {
             for (c, name) in self.keep_caps.iter().zip(&self.keep_cap_names) {

compartments/kryptikd/src/policy/tests.rs

@@ -52,7 +52,7 @@ fn errors_carry_line_number() {
 
 #[test]
 fn dangerous_capabilities_cannot_be_kept() {
-    for name in ["CAP_SYS_ADMIN", "CAP_SYS_PTRACE", "CAP_DAC_OVERRIDE", "CAP_SETUID", "CAP_SYS_MODULE", "CAP_MKNOD"] {
+    for name in ["CAP_SYS_ADMIN", "CAP_SYS_PTRACE", "CAP_DAC_OVERRIDE", "CAP_SETPCAP", "CAP_SYS_MODULE", "CAP_MKNOD"] {
         let err = parse(&format!("keep-capability {name}\n"), "t").unwrap_err();
         assert!(err.to_string().contains("cannot be kept"), "{name}: {err}");
     }
@@ -87,11 +87,31 @@ fn only_nic_zone_keeps_net_caps() {
         assert!(e.to_string().contains("owns the NIC"), "{cap}: {e}");
         assert!(p.check_for_zone(&z("none")).is_err());
     }
+    // dhcpcd's three, for its drop to a user of its own, likewise.
+    let p = parse("keep-capability CAP_SETUID\nkeep-capability CAP_SETGID\nkeep-capability CAP_SYS_CHROOT\n", "t").unwrap();
+    assert!(crate::caps::keeps_privsep(&p.keep_caps));
+    assert!(p.check_for_zone(&z("nic")).is_ok());
+    assert!(p.check_for_zone(&z("routed")).unwrap_err().to_string().contains("owns the NIC"));
+    assert!(p.check_for_zone(&z("none")).is_err());
     // Other keepable capabilities are not mode-restricted.
     let p = parse("keep-capability CAP_SYS_NICE\n", "t").unwrap();
     assert!(p.check_for_zone(&z("routed")).is_ok());
 }
 
+#[test]
+fn privsep_capabilities_kept_together() {
+    for text in [
+        "keep-capability CAP_SETUID\n",
+        "keep-capability CAP_SETGID\nkeep-capability CAP_SYS_CHROOT\n",
+        "keep-capability CAP_SYS_CHROOT\nkeep-capability CAP_NET_ADMIN\n",
+    ] {
+        let err = parse(text, "t").unwrap_err();
+        assert!(err.to_string().contains("kept together or not at all"), "{text:?}: {err}");
+    }
+    let p = parse("keep-capability CAP_SYS_CHROOT\nkeep-capability CAP_SETGID\nkeep-capability CAP_SETUID\n", "t").unwrap();
+    assert_eq!(p.keep_caps.len(), 3);
+}
+
 #[test]
 fn af_netlink_lifts_protocol_check() {
     let p = parse("allow-socket AF_NETLINK\n", "t").unwrap();