From d9a4705a792a7fb5987e7b9c94d8cca1c5e35fa2 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Thu, 9 Jul 2026 19:46:30 -0400 Subject: [PATCH 01/13] Encrypt VLAN/VXLAN tunnel traffic with per-tunnel AES-256 keys Every VLAN/VXLAN link the server builds now gets its own random AES-256 key, sent to both endpoints alongside the existing VlanSetup/VxlanSetup control messages. VLAN traffic is encrypted/decrypted in nullnet-client's userspace forwarder (AES-256-GCM); VXLAN traffic is protected by a per-tunnel kernel IPsec (XFRM/ESP) SA, since that path is handled entirely by the kernel's native vxlan interface. Each VXLAN tunnel also gets its own UDP dstport (replacing the shared 4789 default) so XFRM policies can tell concurrent tunnels between the same host pair apart. --- Cargo.lock | 1 + members/nullnet-client/Cargo.toml | 3 +- members/nullnet-client/src/control_channel.rs | 65 ++++++++-- .../src/craft/reject_payloads.rs | 73 ++++------- members/nullnet-client/src/crypto.rs | 118 ++++++++++++++++++ members/nullnet-client/src/forward/receive.rs | 36 +++++- members/nullnet-client/src/forward/send.rs | 23 +++- members/nullnet-client/src/main.rs | 4 +- members/nullnet-client/src/peers/peer.rs | 17 +++ .../vxlan_scripts/vxlan-setup.sh | 49 ++++++-- .../vxlan_scripts/vxlan-teardown.sh | 23 +++- .../nullnet-grpc-lib/proto/nullnet_grpc.proto | 18 +++ .../src/proto/nullnet_grpc.rs | 24 ++++ members/nullnet-server/src/net.rs | 46 ++++++- members/nullnet-server/src/net_id_pool.rs | 109 ++++++++++++++++ .../nullnet-server/src/nullnet_grpc_impl.rs | 36 +++++- members/nullnet-server/src/orchestrator.rs | 45 ++++++- 17 files changed, 596 insertions(+), 94 deletions(-) create mode 100644 members/nullnet-client/src/crypto.rs diff --git a/Cargo.lock b/Cargo.lock index e04dfcb2..28a34a47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2769,6 +2769,7 @@ dependencies = [ name = "nullnet-client" version = "0.1.0" dependencies = [ + "aes-gcm", "chrono", "clap", "etherparse", diff --git a/members/nullnet-client/Cargo.toml b/members/nullnet-client/Cargo.toml index 79affbb6..0c6fb178 100644 --- a/members/nullnet-client/Cargo.toml +++ b/members/nullnet-client/Cargo.toml @@ -23,4 +23,5 @@ futures = "0.3.32" network-interface = "2.0.5" gag.workspace = true chrono.workspace = true -nfq = "0.2" \ No newline at end of file +nfq = "0.2" +aes-gcm = "0.10" \ No newline at end of file diff --git a/members/nullnet-client/src/control_channel.rs b/members/nullnet-client/src/control_channel.rs index ccfa0588..ba01b631 100644 --- a/members/nullnet-client/src/control_channel.rs +++ b/members/nullnet-client/src/control_channel.rs @@ -162,6 +162,24 @@ async fn handle_vlan_setup( }), ); })?; + // Fail before touching the network if the key is malformed — running + // this tunnel without a valid key would mean forwarding traffic in the + // clear instead of encrypted. + let encryption_key: [u8; 32] = message + .encryption_key + .try_into() + .map_err(|_| "VLAN setup message carried a malformed encryption key") + .handle_err(location!()) + .inspect_err(|e| { + fire_event( + &grpc, + AgentEventKind::VlanSetupFailed(AgentVlanSetupFailed { + vlan_id: message.vlan_id, + local_veth: local_veth.to_string(), + error_reason: e.to_str().to_string(), + }), + ); + })?; // setup VLAN on this machine let init_t = std::time::Instant::now(); @@ -176,11 +194,12 @@ async fn handle_vlan_setup( init_t.elapsed().as_millis() ); - // register peer - peers - .write() - .await - .insert(VethKey::new(remote_veth, vlan_id), remote_ip); + // register peer + this tunnel's encryption key + { + let mut peers = peers.write().await; + peers.insert(VethKey::new(remote_veth, vlan_id), remote_ip); + peers.insert_key(vlan_id, &encryption_key); + } // add host mapping if needed if let Some(host_mapping) = &message.host_mapping { @@ -288,6 +307,25 @@ async fn handle_vxlan_setup( .remote_ip .parse::() .handle_err(location!())?; + // Fail before touching the network if the key is malformed — running + // this tunnel without a valid key would mean forwarding traffic in the + // clear instead of encrypted. + let encryption_key: [u8; 32] = message + .encryption_key + .try_into() + .map_err(|_| "VXLAN setup message carried a malformed encryption key") + .handle_err(location!()) + .inspect_err(|e| { + fire_event( + &grpc, + AgentEventKind::VxlanSetupFailed(AgentVxlanSetupFailed { + vxlan_id, + ns_name: ns_name.clone(), + error_code: -1, + }), + ); + eprintln!("[vxlan_setup] {}", e.to_str()); + })?; // setup VXLAN on this machine (optionally attaching a Docker container) let init_t = std::time::Instant::now(); @@ -298,7 +336,9 @@ async fn handle_vxlan_setup( .arg(br_name) .arg(br_net.to_string()) .arg(local_ip.to_string()) - .arg(remote_ip.to_string()); + .arg(remote_ip.to_string()) + .arg(hex_encode(&encryption_key)) + .arg(message.dstport.to_string()); if let Some(container) = &message.docker_container { cmd.arg(container); } @@ -459,7 +499,12 @@ fn handle_vxlan_teardown( let br_name = message.br_name; let mut cmd = std::process::Command::new("./vxlan_scripts/vxlan-teardown.sh"); - cmd.arg(vxlan_id.to_string()).arg(&ns_name).arg(&br_name); + cmd.arg(vxlan_id.to_string()) + .arg(&ns_name) + .arg(&br_name) + .arg(&message.local_ip) + .arg(&message.remote_ip) + .arg(message.dstport.to_string()); if let Some(container) = &message.docker_container { cmd.arg(container); } @@ -710,3 +755,9 @@ fn remove_hosts_entry(content: &str, name: &str) -> String { .collect(); lines.join("\n") + "\n" } + +/// Lowercase hex encoding, used to pass the tunnel's AES key to +/// `vxlan-setup.sh`/`vxlan-teardown.sh` as a shell argument. +fn hex_encode(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/members/nullnet-client/src/craft/reject_payloads.rs b/members/nullnet-client/src/craft/reject_payloads.rs index 52ddaf05..b2795951 100644 --- a/members/nullnet-client/src/craft/reject_payloads.rs +++ b/members/nullnet-client/src/craft/reject_payloads.rs @@ -3,66 +3,45 @@ use etherparse::{ Icmpv4Header, Icmpv4Type, IpFragOffset, IpNumber, LaxPacketHeaders, LinkExtHeader, LinkHeader, NetHeaders, TcpOptions, TransportHeader, }; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::net::UdpSocket; -/// Sends a proper message to gracefully acknowledge a peer that a packet was rejected, +/// Builds a proper message to gracefully acknowledge a peer that a packet was rejected, /// based on the observed protocol: /// - in case of TCP, a packet with RST and ACK flag is sent /// - in case of UDP, an ICMP port unreachable message is sent /// - in case of other protocols, an ICMP host unreachable message is sent -pub async fn send_termination_message( - packet: &[u8], - socket: &Arc, - remote_socket: SocketAddr, -) { - let Ok(headers) = LaxPacketHeaders::from_ethernet(packet) else { - return; - }; +/// +/// Returns the plaintext Ethernet frame to send; the caller (`forward/receive.rs`) +/// is responsible for encrypting and framing it before it hits the wire, same as +/// any other outbound frame on the VLAN forwarder. +pub fn build_termination_message(packet: &[u8]) -> Option> { + let headers = LaxPacketHeaders::from_ethernet(packet).ok()?; let Some(NetHeaders::Ipv4(ip_header, _)) = &headers.net else { - return; + return None; }; let IpNumber(proto) = ip_header.protocol; match proto { - 6 => Box::pin(send_tcp_rst(headers, socket, remote_socket)).await, + 6 => send_tcp_rst(headers), 17 => { // port unreachable let icmp_type = Icmpv4Type::DestinationUnreachable(DestUnreachableHeader::Port); - Box::pin(send_destination_unreachable( - packet, - headers, - socket, - icmp_type, - remote_socket, - )) - .await; + send_destination_unreachable(packet, headers, icmp_type) } _ => { // host unreachable let icmp_type = Icmpv4Type::DestinationUnreachable(DestUnreachableHeader::Host); - Box::pin(send_destination_unreachable( - packet, - headers, - socket, - icmp_type, - remote_socket, - )) - .await; + send_destination_unreachable(packet, headers, icmp_type) } } } -async fn send_destination_unreachable( +fn send_destination_unreachable( packet: &[u8], headers: LaxPacketHeaders<'_>, - socket: &Arc, icmp_type: Icmpv4Type, - remote_socket: SocketAddr, -) { +) -> Option> { let Some(LinkHeader::Ethernet2(mut ethernet_header)) = headers.link else { - return; + return None; }; std::mem::swap( &mut ethernet_header.source, @@ -80,7 +59,7 @@ async fn send_destination_unreachable( .collect(); let Some(NetHeaders::Ipv4(mut ip_header, _)) = headers.net else { - return; + return None; }; let original_ip_header_bytes = ip_header.to_bytes(); let size_up_to_ip_header = @@ -112,19 +91,12 @@ async fn send_destination_unreachable( &icmp_payload[..], ].concat(); - socket - .send_to(&pkt_response, remote_socket) - .await - .unwrap_or(0); + Some(pkt_response) } -async fn send_tcp_rst( - headers: LaxPacketHeaders<'_>, - socket: &Arc, - remote_socket: SocketAddr, -) { +fn send_tcp_rst(headers: LaxPacketHeaders<'_>) -> Option> { let Some(LinkHeader::Ethernet2(mut ethernet_header)) = headers.link else { - return; + return None; }; std::mem::swap( &mut ethernet_header.source, @@ -142,7 +114,7 @@ async fn send_tcp_rst( .collect(); let Some(NetHeaders::Ipv4(mut ip_header, _)) = headers.net else { - return; + return None; }; ip_header.identification = 0; ip_header.fragment_offset = IpFragOffset::ZERO; @@ -152,7 +124,7 @@ async fn send_tcp_rst( let ip_header_bytes = ip_header.to_bytes(); let Some(TransportHeader::Tcp(mut tcp_header)) = headers.transport else { - return; + return None; }; let src_port_orig = tcp_header.source_port; let seq_num_orig = tcp_header.sequence_number; @@ -187,8 +159,5 @@ async fn send_tcp_rst( &tcp_header_bytes[..], ].concat(); - socket - .send_to(&pkt_response, remote_socket) - .await - .unwrap_or(0); + Some(pkt_response) } diff --git a/members/nullnet-client/src/crypto.rs b/members/nullnet-client/src/crypto.rs new file mode 100644 index 00000000..537bc360 --- /dev/null +++ b/members/nullnet-client/src/crypto.rs @@ -0,0 +1,118 @@ +use aes_gcm::aead::rand_core::RngCore; +use aes_gcm::aead::{Aead, KeyInit, OsRng}; +use aes_gcm::{Aes256Gcm, Key, Nonce}; + +/// AES-256-GCM cipher for one VLAN tunnel's traffic. Wire format produced by +/// [`Self::encrypt`] / consumed by [`Self::decrypt`] is `nonce[12] || ciphertext+tag`. +/// Unlike `nullnet-server`'s cert-at-rest `Encryptor` (keyed once, process-wide, +/// operates on UTF-8 strings), this is constructed per-tunnel from the key the +/// server hands out at `VlanSetup` time and operates on raw Ethernet frames. +pub(crate) struct TunnelCipher { + cipher: Aes256Gcm, +} + +impl TunnelCipher { + pub(crate) fn new(key: &[u8; 32]) -> Self { + Self { + cipher: Aes256Gcm::new(Key::::from_slice(key)), + } + } + + pub(crate) fn encrypt(&self, plaintext: &[u8]) -> Option> { + let mut nonce_bytes = [0u8; 12]; + OsRng.fill_bytes(&mut nonce_bytes); + let mut out = nonce_bytes.to_vec(); + out.extend( + self.cipher + .encrypt(Nonce::from_slice(&nonce_bytes), plaintext) + .ok()?, + ); + Some(out) + } + + pub(crate) fn decrypt(&self, data: &[u8]) -> Option> { + if data.len() < 12 { + return None; + } + let (nonce_bytes, ciphertext) = data.split_at(12); + self.cipher + .decrypt(Nonce::from_slice(nonce_bytes), ciphertext) + .ok() + } +} + +/// Wire framing for the VLAN userspace forwarder (`forward/send.rs`, +/// `forward/receive.rs`): every datagram on the forward socket is +/// `vlan_id[2, big-endian] || nonce[12] || ciphertext+tag`. The vlan_id has +/// to be readable in the clear so the receiver knows which tunnel's key to +/// decrypt with before it can read anything else. +pub(crate) fn seal(vlan_id: u16, cipher: &TunnelCipher, plaintext: &[u8]) -> Option> { + let mut out = vlan_id.to_be_bytes().to_vec(); + out.extend(cipher.encrypt(plaintext)?); + Some(out) +} + +/// Splits a raw forward-socket datagram into its cleartext `vlan_id` and the +/// remaining `nonce || ciphertext+tag` slice, ready for `TunnelCipher::decrypt`. +pub(crate) fn open_vlan_id(datagram: &[u8]) -> Option<(u16, &[u8])> { + if datagram.len() < 2 { + return None; + } + let (vlan_id_bytes, rest) = datagram.split_at(2); + Some(( + u16::from_be_bytes([vlan_id_bytes[0], vlan_id_bytes[1]]), + rest, + )) +} + +#[cfg(test)] +mod tests { + use super::{TunnelCipher, open_vlan_id, seal}; + + #[test] + fn round_trip() { + let cipher = TunnelCipher::new(&[7u8; 32]); + let frame = b"pretend this is an ethernet frame"; + let ct = cipher.encrypt(frame).unwrap(); + assert_ne!(ct, frame); + assert_eq!(cipher.decrypt(&ct).unwrap(), frame); + } + + #[test] + fn nonce_randomizes_ciphertext() { + let cipher = TunnelCipher::new(&[7u8; 32]); + assert_ne!( + cipher.encrypt(b"same").unwrap(), + cipher.encrypt(b"same").unwrap() + ); + } + + #[test] + fn wrong_key_fails() { + let ct = TunnelCipher::new(&[1u8; 32]).encrypt(b"secret").unwrap(); + assert!(TunnelCipher::new(&[2u8; 32]).decrypt(&ct).is_none()); + } + + #[test] + fn truncated_data_fails_without_panicking() { + let cipher = TunnelCipher::new(&[3u8; 32]); + assert!(cipher.decrypt(&[0u8; 5]).is_none()); + } + + #[test] + fn seal_then_open_round_trips_vlan_id_and_plaintext() { + let cipher = TunnelCipher::new(&[9u8; 32]); + let frame = b"ethernet frame payload"; + let datagram = seal(4242, &cipher, frame).unwrap(); + + let (vlan_id, rest) = open_vlan_id(&datagram).unwrap(); + assert_eq!(vlan_id, 4242); + assert_eq!(cipher.decrypt(rest).unwrap(), frame); + } + + #[test] + fn open_vlan_id_rejects_short_datagrams() { + assert!(open_vlan_id(&[0u8]).is_none()); + assert!(open_vlan_id(&[]).is_none()); + } +} diff --git a/members/nullnet-client/src/forward/receive.rs b/members/nullnet-client/src/forward/receive.rs index ad90662a..bc53758a 100644 --- a/members/nullnet-client/src/forward/receive.rs +++ b/members/nullnet-client/src/forward/receive.rs @@ -5,8 +5,10 @@ use tokio::net::UdpSocket; use tokio::sync::RwLock; use tun_rs::AsyncDevice; -use crate::craft::reject_payloads::send_termination_message; +use crate::craft::reject_payloads::build_termination_message; +use crate::crypto; use crate::forward::frame::Frame; +use crate::peers::peer::Peers; /// Handles incoming network packets (receives packets from the socket and sends them to the TAP interface), /// ensuring the firewall rules are correctly observed. @@ -14,29 +16,51 @@ pub async fn receive( device: &Arc, socket: &Arc, firewall: &Arc>, + peers: &Arc>, ) { let mut frame = Frame::new(); let mut remote_socket; loop { - // wait until there is an incoming packet on the socket (packets on the socket are raw IP) + // wait until there is an incoming datagram on the socket let Ok((s, r)) = socket.recv_from(&mut frame.frame).await else { continue; }; (frame.size, remote_socket) = (s, r); if frame.size > 0 { - let pkt_data = frame.pkt_data(); + let datagram = &frame.frame[..frame.size]; + // the vlan_id has to be readable before decryption so we know + // which tunnel's key to decrypt with + let Some((vlan_id, sealed)) = crypto::open_vlan_id(datagram) else { + continue; + }; + let Some(cipher) = peers.read().await.get_key(vlan_id) else { + continue; + }; + // decrypt as the packet exits the tunnel; auth failure (wrong + // key, corrupted/spoofed datagram) drops it here + let Some(pkt_data) = cipher.decrypt(sealed) else { + continue; + }; + match firewall .read() .await - .resolve_packet(pkt_data, FirewallDirection::IN) + .resolve_packet(&pkt_data, FirewallDirection::IN) { FirewallAction::ACCEPT => { // write packet to the kernel - device.send(pkt_data).await.unwrap_or(0); + device.send(&pkt_data).await.unwrap_or(0); } FirewallAction::REJECT => { - send_termination_message(pkt_data, socket, remote_socket).await; + if let Some(reply) = build_termination_message(&pkt_data) + && let Some(reply_datagram) = crypto::seal(vlan_id, &cipher, &reply) + { + socket + .send_to(&reply_datagram, remote_socket) + .await + .unwrap_or(0); + } } FirewallAction::DENY => {} } diff --git a/members/nullnet-client/src/forward/send.rs b/members/nullnet-client/src/forward/send.rs index 2854580b..1ebf27ce 100644 --- a/members/nullnet-client/src/forward/send.rs +++ b/members/nullnet-client/src/forward/send.rs @@ -7,6 +7,7 @@ use tokio::net::UdpSocket; use tokio::sync::RwLock; use tun_rs::AsyncDevice; +use crate::crypto; use crate::forward::frame::Frame; use crate::peers::peer::{Peers, VethKey}; @@ -26,7 +27,7 @@ pub async fn send( if frame.size > 0 { // send the packet to the socket let pkt_data = frame.pkt_data(); - let Ok(dst_socket) = get_dst_socket(pkt_data, &peers).await else { + let Ok((dst_socket, vlan_id)) = get_dst_socket(pkt_data, &peers).await else { continue; }; match firewall @@ -35,7 +36,15 @@ pub async fn send( .resolve_packet(pkt_data, FirewallDirection::OUT) { FirewallAction::ACCEPT => { - socket.send_to(pkt_data, dst_socket).await.unwrap_or(0); + // encrypt as the packet enters the tunnel: without a key + // for this vlan_id (tunnel torn down mid-flight, or + // setup never landed) there is nothing safe to send. + let Some(cipher) = peers.read().await.get_key(vlan_id) else { + continue; + }; + if let Some(datagram) = crypto::seal(vlan_id, &cipher, pkt_data) { + socket.send_to(&datagram, dst_socket).await.unwrap_or(0); + } } FirewallAction::DENY | FirewallAction::REJECT => {} } @@ -43,7 +52,10 @@ pub async fn send( } } -async fn get_dst_socket(pkt_data: &[u8], peers: &Arc>) -> Result { +async fn get_dst_socket( + pkt_data: &[u8], + peers: &Arc>, +) -> Result<(SocketAddr, u16), Error> { let headers = LaxPacketHeaders::from_ethernet(pkt_data).handle_err(location!())?; let vlan_id = headers .vlan_ids() @@ -63,10 +75,11 @@ async fn get_dst_socket(pkt_data: &[u8], peers: &Arc>) -> Result, + /// Per-tunnel AES-256-GCM cipher, keyed by `vlan_id`. Populated from the + /// key the server hands out in `VlanSetup` and used by the userspace + /// forwarder (`forward/send.rs`, `forward/receive.rs`) to encrypt/decrypt + /// this tunnel's traffic. `Arc`-wrapped so callers can clone a handle out + /// without holding the `Peers` lock across the actual crypto work. + keys: HashMap>, } impl Peers { @@ -24,8 +32,17 @@ impl Peers { self.ips.insert(veth_key, eth_ip); } + pub fn insert_key(&mut self, vlan_id: u16, key: &[u8; 32]) { + self.keys.insert(vlan_id, Arc::new(TunnelCipher::new(key))); + } + + pub fn get_key(&self, vlan_id: u16) -> Option> { + self.keys.get(&vlan_id).cloned() + } + pub fn remove(&mut self, vlan_id: u16) { self.ips.retain(|key, _| key.vlan_id != vlan_id); + self.keys.remove(&vlan_id); } } diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 366b7340..50047e85 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -1,10 +1,10 @@ #!/bin/bash # Read CLI arguments: -if [ "$#" -lt 7 ] || [ "$#" -gt 8 ]; then - echo "Usage: $0 [docker_container]" - echo "Example (standalone): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104" - echo "Example (docker): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104 my_container" +if [ "$#" -lt 9 ] || [ "$#" -gt 10 ]; then + echo "Usage: $0 [docker_container]" + echo "Example (standalone): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104 <64 hex chars> 20100" + echo "Example (docker): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104 <64 hex chars> 20100 my_container" exit 1 fi @@ -15,7 +15,9 @@ BR_NAME=$4 BR_NET=$5 LOCAL_IP=$6 REMOTE_IP=$7 -DOCKER_CONTAINER=$8 +KEY_HEX=$8 +DSTPORT=$9 +DOCKER_CONTAINER=${10} BR_IP=$(echo $BR_NET | cut -d'/' -f1) @@ -56,7 +58,9 @@ if [ -z "$DOCKER_CONTAINER" ]; then fi if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then - # Same host: connect bridges with a veth pair instead of a VXLAN tunnel + # Same host: connect bridges with a veth pair instead of a VXLAN tunnel. + # Traffic never leaves the host, so there's nothing to encrypt here — + # no XFRM setup on this branch. VETH_S="veth-${VXLAN_ID}-s" VETH_C="veth-${VXLAN_ID}-c" # Both ends are created atomically; the losing task's EEXIST is harmless @@ -70,11 +74,40 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then sudo ip link set "$LOCAL_VETH" master "$BR_NAME" sudo ip link set "$LOCAL_VETH" mtu $OVERLAY_MTU up else - # Create the VXLAN tunnel using your physical IP and interface: - sudo ip link add vxlan-$NS_NAME type vxlan id $VXLAN_ID local $LOCAL_IP remote $REMOTE_IP dstport 4789 # dev ens18 + # Create the VXLAN tunnel using your physical IP and interface. Each + # tunnel gets its own dstport (instead of the IANA-standard 4789) so + # the XFRM policies below can tell concurrent tunnels between the same + # host pair apart. + sudo ip link add vxlan-$NS_NAME type vxlan id $VXLAN_ID local $LOCAL_IP remote $REMOTE_IP dstport $DSTPORT # dev ens18 # Attach the VXLAN to the bridge: sudo ip link set vxlan-$NS_NAME master $BR_NAME sudo ip link set vxlan-$NS_NAME mtu $OVERLAY_MTU up + + # Encrypt this tunnel's traffic at the kernel level (AES-256-GCM via + # IPsec/ESP, transport mode) between the two hosts' physical IPs, + # scoped to this tunnel's dstport so it doesn't collide with any other + # concurrent VXLAN tunnel between the same host pair. + # + # RFC4106 GCM keys are "AES key || 4-byte salt". The server only hands + # out a 32-byte AES key (shared verbatim by both VLAN's software AEAD + # and this XFRM SA), so the salt is derived here, identically on both + # ends, from that same key — it doesn't need to be secret on its own, + # only reproducible from the shared secret both sides already have. + SALT_HEX=$(printf '%s' "$KEY_HEX" | sha256sum | cut -c1-8) + AEAD_KEY_HEX="${KEY_HEX}${SALT_HEX}" + SPI=$(printf '0x%08x' "$VXLAN_ID") + + # Outbound: this host -> remote. + sudo ip xfrm state add src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI \ + mode transport aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 + sudo ip xfrm policy add src $LOCAL_IP dst $REMOTE_IP dir out proto udp dport $DSTPORT \ + tmpl src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI mode transport + + # Inbound: remote -> this host. + sudo ip xfrm state add src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI \ + mode transport aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 + sudo ip xfrm policy add src $REMOTE_IP dst $LOCAL_IP dir in proto udp dport $DSTPORT \ + tmpl src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI mode transport fi # Enable IP forwarding: diff --git a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh index 49a938aa..99e9d25d 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh @@ -1,17 +1,30 @@ #!/bin/bash # Read CLI arguments: -if [ "$#" -lt 3 ] || [ "$#" -gt 4 ]; then - echo "Usage: $0 [docker_container]" - echo "Example (standalone): $0 100 ns_100_s br_100_s" - echo "Example (docker): $0 100 ns_100_s br_100_s my_container" +if [ "$#" -lt 6 ] || [ "$#" -gt 7 ]; then + echo "Usage: $0 [docker_container]" + echo "Example (standalone): $0 100 ns_100_s br_100_s 192.168.1.102 192.168.1.104 20100" + echo "Example (docker): $0 100 ns_100_s br_100_s 192.168.1.102 192.168.1.104 20100 my_container" exit 1 fi VXLAN_ID=$1 NS_NAME=$2 BR_NAME=$3 -DOCKER_CONTAINER=$4 +LOCAL_IP=$4 +REMOTE_IP=$5 +DSTPORT=$6 +DOCKER_CONTAINER=$7 + +# Remove this tunnel's XFRM state + policy pair, if any was installed (the +# same-host branch of vxlan-setup.sh never creates one). +if [ "$LOCAL_IP" != "$REMOTE_IP" ]; then + SPI=$(printf '0x%08x' "$VXLAN_ID") + sudo ip xfrm policy delete src $LOCAL_IP dst $REMOTE_IP dir out proto udp dport $DSTPORT 2>/dev/null + sudo ip xfrm state delete src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI 2>/dev/null + sudo ip xfrm policy delete src $REMOTE_IP dst $LOCAL_IP dir in proto udp dport $DSTPORT 2>/dev/null + sudo ip xfrm state delete src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI 2>/dev/null +fi # Remove the VXLAN tunnel or same-host veth pair: sudo ip link set vxlan-$NS_NAME down && sudo ip link del vxlan-$NS_NAME diff --git a/members/nullnet-grpc-lib/proto/nullnet_grpc.proto b/members/nullnet-grpc-lib/proto/nullnet_grpc.proto index b7f3e88f..3df1cd8f 100644 --- a/members/nullnet-grpc-lib/proto/nullnet_grpc.proto +++ b/members/nullnet-grpc-lib/proto/nullnet_grpc.proto @@ -90,6 +90,10 @@ message VlanSetup { string local_ip = 5; string remote_ip = 6; optional HostMapping host_mapping = 7; + // Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + // sent identically to both endpoints. Used to encrypt/decrypt traffic in + // the client's userspace VLAN forwarder (see forward/send.rs, forward/receive.rs). + bytes encryption_key = 8; } message VlanTeardown { @@ -111,6 +115,15 @@ message VxlanSetup { // The receiving client installs DNAT(dnat_port -> overlay_ip) so the // initiator's traffic on that local port is steered into the new VXLAN. optional uint32 dnat_port = 11; + // Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + // sent identically to both endpoints. Used as the XFRM/ESP SA key that + // encrypts this tunnel's traffic at the kernel level. + bytes encryption_key = 12; + // Per-tunnel VXLAN UDP destination port (replaces the IANA-standard 4789 + // default). Each tunnel gets a distinct port so an XFRM policy — which + // selects by src/dst IP and port, not by VNI — can tell concurrent + // tunnels between the same host pair apart. + uint32 dstport = 13; } message VxlanTeardown { @@ -118,6 +131,11 @@ message VxlanTeardown { string ns_name = 2; string br_name = 3; optional string docker_container = 4; + // local_ip/remote_ip/dstport: needed to remove this tunnel's XFRM SA + + // policy pair (same values used to install them in VxlanSetup). + string local_ip = 5; + string remote_ip = 6; + uint32 dstport = 7; } message MsgId { diff --git a/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs b/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs index 8a2d87d4..d38a50c0 100644 --- a/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs +++ b/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs @@ -63,6 +63,11 @@ pub struct VlanSetup { pub remote_ip: ::prost::alloc::string::String, #[prost(message, optional, tag = "7")] pub host_mapping: ::core::option::Option, + /// Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + /// sent identically to both endpoints. Used to encrypt/decrypt traffic in + /// the client's userspace VLAN forwarder (see forward/send.rs, forward/receive.rs). + #[prost(bytes = "vec", tag = "8")] + pub encryption_key: ::prost::alloc::vec::Vec, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct VlanTeardown { @@ -96,6 +101,17 @@ pub struct VxlanSetup { /// initiator's traffic on that local port is steered into the new VXLAN. #[prost(uint32, optional, tag = "11")] pub dnat_port: ::core::option::Option, + /// Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + /// sent identically to both endpoints. Used as the XFRM/ESP SA key that + /// encrypts this tunnel's traffic at the kernel level. + #[prost(bytes = "vec", tag = "12")] + pub encryption_key: ::prost::alloc::vec::Vec, + /// Per-tunnel VXLAN UDP destination port (replaces the IANA-standard 4789 + /// default). Each tunnel gets a distinct port so an XFRM policy — which + /// selects by src/dst IP and port, not by VNI — can tell concurrent + /// tunnels between the same host pair apart. + #[prost(uint32, tag = "13")] + pub dstport: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct VxlanTeardown { @@ -107,6 +123,14 @@ pub struct VxlanTeardown { pub br_name: ::prost::alloc::string::String, #[prost(string, optional, tag = "4")] pub docker_container: ::core::option::Option<::prost::alloc::string::String>, + /// local_ip/remote_ip/dstport: needed to remove this tunnel's XFRM SA + + /// policy pair (same values used to install them in VxlanSetup). + #[prost(string, tag = "5")] + pub local_ip: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub remote_ip: ::prost::alloc::string::String, + #[prost(uint32, tag = "7")] + pub dstport: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MsgId { diff --git a/members/nullnet-server/src/net.rs b/members/nullnet-server/src/net.rs index 726baba4..fe6d63fb 100644 --- a/members/nullnet-server/src/net.rs +++ b/members/nullnet-server/src/net.rs @@ -17,9 +17,20 @@ pub(crate) trait NetExt { remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: Option, ) -> Option<(Ipv4Addr, NetMessage)>; - fn teardown(self, net_id: u32, side: &str, docker_container: Option) -> NetMessage; + #[allow(clippy::too_many_arguments)] + fn teardown( + self, + net_id: u32, + side: &str, + docker_container: Option, + local_ip: IpAddr, + remote_ip: IpAddr, + dstport: Option, + ) -> NetMessage; } impl NetExt for Net { @@ -32,9 +43,18 @@ impl NetExt for Net { remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: Option, ) -> Option<(Ipv4Addr, NetMessage)> { match self { - Net::Vlan => vlan_setup(msg_id, dest, remote_server_name, net_id, remote), + Net::Vlan => vlan_setup( + msg_id, + dest, + remote_server_name, + net_id, + remote, + encryption_key, + ), Net::Vxlan => vxlan_setup( msg_id, dest, @@ -43,11 +63,21 @@ impl NetExt for Net { remote, docker_containers, dnat_port, + encryption_key, + dstport.unwrap_or(0), ), } } - fn teardown(self, net_id: u32, side: &str, docker_container: Option) -> NetMessage { + fn teardown( + self, + net_id: u32, + side: &str, + docker_container: Option, + local_ip: IpAddr, + remote_ip: IpAddr, + dstport: Option, + ) -> NetMessage { match self { Net::Vlan => NetMessage { message: Some(net_message::Message::VlanTeardown(VlanTeardown { @@ -60,6 +90,9 @@ impl NetExt for Net { ns_name: format!("ns_{net_id}_{side}"), br_name: format!("br_{net_id}_{side}"), docker_container, + local_ip: local_ip.to_string(), + remote_ip: remote_ip.to_string(), + dstport: u32::from(dstport.unwrap_or(0)), })), }, } @@ -73,6 +106,7 @@ fn vlan_setup( remote_server_name: Option, vlan_id: u32, remote: IpAddr, + encryption_key: [u8; 32], ) -> Option<(Ipv4Addr, NetMessage)> { // Map vlan_id to a /30 block within 10.0.0.0/8. // Each ID gets 4 IPs (2 usable), with 2 IPs used for server/client veth. @@ -106,11 +140,13 @@ fn vlan_setup( local_ip: dest.to_string(), remote_ip: remote.to_string(), host_mapping, + encryption_key: encryption_key.to_vec(), })), }, )) } +#[allow(clippy::too_many_arguments)] fn vxlan_setup( msg_id: String, dest: IpAddr, @@ -119,6 +155,8 @@ fn vxlan_setup( remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: u32, ) -> Option<(Ipv4Addr, NetMessage)> { // Map vxlan_id to a /29 block within 10.0.0.0/8. // Each ID gets 8 IPs (6 usable), with 4 IPs used for ns/br server/client. @@ -176,6 +214,8 @@ fn vxlan_setup( host_mapping, docker_container, dnat_port, + encryption_key: encryption_key.to_vec(), + dstport, })), }, )) diff --git a/members/nullnet-server/src/net_id_pool.rs b/members/nullnet-server/src/net_id_pool.rs index 291fa22c..e17f5076 100644 --- a/members/nullnet-server/src/net_id_pool.rs +++ b/members/nullnet-server/src/net_id_pool.rs @@ -1,3 +1,5 @@ +use aes_gcm::aead::OsRng; +use aes_gcm::aead::rand_core::RngCore; use std::collections::BTreeSet; use std::sync::LazyLock; @@ -68,6 +70,61 @@ impl NetIdPool { } } +/// Minimum/maximum allocatable UDP port for per-tunnel VXLAN dstports. +/// Kept out of the IANA ephemeral range (32768-60999) and away from 4789 +/// (the VXLAN default) to avoid colliding with unrelated local sockets. +const MIN_VXLAN_PORT: u16 = 20000; +const MAX_VXLAN_PORT: u16 = 60000; + +/// Pool of per-tunnel UDP destination ports, used so concurrent VXLAN +/// tunnels between the same physical host pair each get a distinct dstport. +/// This is what lets an XFRM policy (which selects by IP + port, not VNI) +/// tell those tunnels apart. Same allocate/free-with-reuse shape as `NetIdPool`. +#[derive(Debug)] +pub(crate) struct UdpPortPool { + next_fresh: u16, + freed: BTreeSet, +} + +impl UdpPortPool { + pub(crate) fn new() -> Self { + Self { + next_fresh: MIN_VXLAN_PORT, + freed: BTreeSet::new(), + } + } + + pub(crate) fn allocate(&mut self) -> Option { + if let Some(&port) = self.freed.iter().next() { + self.freed.remove(&port); + return Some(port); + } + + if self.next_fresh <= MAX_VXLAN_PORT { + let port = self.next_fresh; + self.next_fresh += 1; + Some(port) + } else { + None + } + } + + pub(crate) fn free(&mut self, port: u16) { + if (MIN_VXLAN_PORT..=MAX_VXLAN_PORT).contains(&port) { + self.freed.insert(port); + } + } +} + +/// Generate a fresh random 32-byte AES-256 key for one tunnel. Called once +/// per net_id allocation; the same bytes are sent to both endpoints so they +/// share a single symmetric key for that tunnel only. +pub(crate) fn generate_key() -> [u8; 32] { + let mut key = [0u8; 32]; + OsRng.fill_bytes(&mut key); + key +} + #[cfg(test)] impl NetIdPool { /// Number of IDs currently in use (allocated but not freed). @@ -187,4 +244,56 @@ mod tests { let free = total - in_use; assert_eq!(total, in_use + free); } + + #[test] + fn test_udp_port_pool_allocate_sequential() { + let mut pool = UdpPortPool::new(); + assert_eq!(pool.allocate(), Some(MIN_VXLAN_PORT)); + assert_eq!(pool.allocate(), Some(MIN_VXLAN_PORT + 1)); + assert_eq!(pool.allocate(), Some(MIN_VXLAN_PORT + 2)); + } + + #[test] + fn test_udp_port_pool_reuse_freed() { + let mut pool = UdpPortPool::new(); + let p1 = pool.allocate().unwrap(); + let p2 = pool.allocate().unwrap(); + pool.allocate(); + + pool.free(p2); + pool.free(p1); + + assert_eq!(pool.allocate(), Some(p1)); + assert_eq!(pool.allocate(), Some(p2)); + } + + #[test] + fn test_udp_port_pool_exhaustion() { + let mut pool = UdpPortPool::new(); + pool.next_fresh = MAX_VXLAN_PORT; + + assert_eq!(pool.allocate(), Some(MAX_VXLAN_PORT)); + assert_eq!(pool.allocate(), None); + + pool.free(MAX_VXLAN_PORT); + assert_eq!(pool.allocate(), Some(MAX_VXLAN_PORT)); + assert_eq!(pool.allocate(), None); + } + + #[test] + fn test_udp_port_pool_free_ignores_out_of_range() { + let mut pool = UdpPortPool::new(); + pool.free(0); + pool.free(MIN_VXLAN_PORT - 1); + pool.free(MAX_VXLAN_PORT + 1); + assert!(pool.freed.is_empty()); + } + + #[test] + fn test_generate_key_is_random_and_full_length() { + let k1 = generate_key(); + let k2 = generate_key(); + assert_eq!(k1.len(), 32); + assert_ne!(k1, k2); + } } diff --git a/members/nullnet-server/src/nullnet_grpc_impl.rs b/members/nullnet-server/src/nullnet_grpc_impl.rs index 935f4fdf..0215862f 100644 --- a/members/nullnet-server/src/nullnet_grpc_impl.rs +++ b/members/nullnet-server/src/nullnet_grpc_impl.rs @@ -1,6 +1,7 @@ use crate::env::NET_TYPE; use crate::events::Event; use crate::graphviz::generate_graphviz; +use crate::net_id_pool::generate_key; use crate::orchestrator::Orchestrator; use crate::services::changes::{ apply_changes, collect_dep_chain_edges, detect_services_list_changes, @@ -12,9 +13,9 @@ use crate::services::service_info::{ServiceInfo, backend_involved_services}; use crate::timeout::check_timeouts; use nullnet_grpc_lib::nullnet_grpc::nullnet_grpc_server::NullnetGrpc; use nullnet_grpc_lib::nullnet_grpc::{ - AgentEvent, BackendTriggerRequest, CertBundle, Empty, MsgId, NetMessage, NetType, PortMapping, - PortMappingBundle, ProxyRequest, ServiceTrigger, Services, ServicesListResponse, Upstream, - agent_event::Event as AgentEventKind, + AgentEvent, BackendTriggerRequest, CertBundle, Empty, MsgId, Net, NetMessage, NetType, + PortMapping, PortMappingBundle, ProxyRequest, ServiceTrigger, Services, ServicesListResponse, + Upstream, agent_event::Event as AgentEventKind, }; use nullnet_liberror::{Error, ErrorHandler, Location, location}; use std::collections::{HashMap, HashSet}; @@ -868,6 +869,31 @@ impl NullnetGrpcImpl { .await; } + // One AES-256 key per tunnel, handed identically to both + // endpoints below. For VXLAN, also reserve a per-tunnel UDP + // dstport so the two hosts' XFRM policies can tell this + // tunnel apart from any other concurrent tunnel between the + // same physical host pair. + let encryption_key = generate_key(); + let dstport = if *NET_TYPE == Net::Vxlan { + match orchestrator.allocate_vxlan_port(net_id).await { + Some(port) => Some(u32::from(port)), + None => { + eprintln!("UDP port pool exhausted"); + orchestrator.free_net_id(net_id).await; + if let Some(stack_map) = services.write().await.get_mut(&stack) + && let Some(ServiceInfo::Registered(reg)) = + stack_map.get_mut(server.name()) + { + reg.remove_client(&client); + } + return EdgeOutcome::Failed; + } + } + } else { + None + }; + let orch = orchestrator.clone(); let cd = client_docker.clone(); let sd = server_docker.clone(); @@ -878,6 +904,8 @@ impl NullnetGrpcImpl { client_ethernet, (cd, sd), None, + encryption_key, + dstport, ); let orch2 = orchestrator.clone(); let cd = client_docker.clone(); @@ -889,6 +917,8 @@ impl NullnetGrpcImpl { server_ethernet, (cd, sd), backend_entry_port, + encryption_key, + dstport, ); let (server_ok, client_ok) = tokio::join!(server_res, client_res); diff --git a/members/nullnet-server/src/orchestrator.rs b/members/nullnet-server/src/orchestrator.rs index 446c09f0..ad4d7e4c 100644 --- a/members/nullnet-server/src/orchestrator.rs +++ b/members/nullnet-server/src/orchestrator.rs @@ -1,7 +1,7 @@ use crate::env::NET_TYPE; use crate::events::{Event, EventStore}; use crate::net::NetExt; -use crate::net_id_pool::NetIdPool; +use crate::net_id_pool::{NetIdPool, UdpPortPool}; use crate::services::changes::{apply_changes, detect_node_disconnect_changes}; use crate::services::input::StackMap; use nullnet_grpc_lib::nullnet_grpc::{ @@ -23,6 +23,12 @@ pub struct Orchestrator { clients: Arc>>, pending: Arc>>>, net_id_pool: Arc>, + /// Per-tunnel VXLAN UDP dstport pool. Unused in VLAN mode. + udp_port_pool: Arc>, + /// net_id -> allocated dstport, for VXLAN tunnels only. Lets + /// `send_net_teardown` free the port without every call site having to + /// thread it through. + net_id_ports: Arc>>, pub(crate) events: EventStore, } @@ -32,6 +38,8 @@ impl Orchestrator { clients: Arc::new(RwLock::new(HashMap::new())), pending: Arc::new(Mutex::new(HashMap::new())), net_id_pool: Arc::new(Mutex::new(NetIdPool::new())), + udp_port_pool: Arc::new(Mutex::new(UdpPortPool::new())), + net_id_ports: Arc::new(Mutex::new(HashMap::new())), events: EventStore::new(), } } @@ -99,6 +107,7 @@ impl Orchestrator { } } + #[allow(clippy::too_many_arguments)] pub(crate) async fn send_net_setup( &self, dest: IpAddr, @@ -107,6 +116,8 @@ impl Orchestrator { remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: Option, ) -> Option { let outbound = self.clients.read().await.get(&dest).cloned(); if let Some(outbound) = outbound { @@ -122,6 +133,8 @@ impl Orchestrator { remote, docker_containers, dnat_port, + encryption_key, + dstport, )?; if outbound.send(Ok(message)).await.is_err() { @@ -197,6 +210,22 @@ impl Orchestrator { self.net_id_pool.lock().await.allocate() } + /// Release a `net_id` that was allocated but never dispatched to either + /// endpoint (e.g. a follow-up allocation failed). No teardown messages + /// are sent — nothing was ever set up on either client. + pub(crate) async fn free_net_id(&self, net_id: u32) { + self.net_id_pool.lock().await.free(net_id); + } + + /// Allocate a per-tunnel VXLAN dstport and remember it against `net_id` + /// so `send_net_teardown` can free it later without the caller having to + /// carry it around. Only meaningful when `NET_TYPE == Net::Vxlan`. + pub(crate) async fn allocate_vxlan_port(&self, net_id: u32) -> Option { + let port = self.udp_port_pool.lock().await.allocate()?; + self.net_id_ports.lock().await.insert(net_id, port); + Some(port) + } + pub(crate) async fn connected_node_ips(&self) -> Vec { self.clients.read().await.keys().copied().collect() } @@ -213,17 +242,27 @@ impl Orchestrator { server_docker: Option, net_id: u32, ) { - for (dest, side, docker) in [(client, "c", client_docker), (server, "s", server_docker)] { + // Peeked (not removed yet) so both teardown messages can carry the + // same dstport that was used to install this tunnel's XFRM state; + // the pool slot itself is freed below, after both sides are notified. + let dstport = self.net_id_ports.lock().await.get(&net_id).copied(); + for (dest, remote, side, docker) in [ + (client, server, "c", client_docker), + (server, client, "s", server_docker), + ] { let outbound = self.clients.read().await.get(&dest).cloned(); if let Some(outbound) = outbound { println!("Sending network {net_id} teardown to client {dest}"); - let message = NET_TYPE.teardown(net_id, side, docker); + let message = NET_TYPE.teardown(net_id, side, docker, dest, remote, dstport); let _ = outbound.send(Ok(message)).await.handle_err(location!()); } } self.net_id_pool.lock().await.free(net_id); + if let Some(port) = self.net_id_ports.lock().await.remove(&net_id) { + self.udp_port_pool.lock().await.free(port); + } } } From 10cd0691beb497f3833c1bfcff000a0024efec33 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Thu, 9 Jul 2026 20:51:06 -0400 Subject: [PATCH 02/13] Encrypt same-host VLAN/VXLAN traffic too, for defense-in-depth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, tunnels whose two endpoints happened to be colocated on the same physical host got no encryption at all: VXLAN's same-host branch used a plain veth pair with no IPsec, and VLAN traffic between two same-host access ports was switched directly by OVS without ever transiting the encrypting userspace forwarder. Neither is reachable by a network sniffer, but both are readable by anything else with sufficient privilege on that same host (a differently-privileged container, a host-level process) — which matters for a project whose whole premise is not trusting other things on the network by default. VXLAN: wrap the same-host veth pair in MACsec (802.1AE, GCM-AES-256), keyed with the same per-tunnel key already delivered for the cross-host IPsec path. Cascades away on teardown when the underlying veth is deleted; also deleted explicitly for clarity. VLAN: replace the single default-normal OVS flow with two more specific ones — traffic arriving from the trunk (already decrypted) is delivered by normal switching; traffic arriving from any access port always exits via the trunk, never directly to another access port. This forces every packet through the TAP and therefore through the existing encrypt/decrypt path in forward/send.rs and forward/receive.rs, with no changes needed there. --- members/nullnet-client/src/commands/mod.rs | 15 +++++--- members/nullnet-client/src/commands/ovs.rs | 35 ++++++++++++++---- .../vxlan_scripts/vxlan-setup.sh | 36 ++++++++++++++++--- .../vxlan_scripts/vxlan-teardown.sh | 8 ++++- 4 files changed, 79 insertions(+), 15 deletions(-) diff --git a/members/nullnet-client/src/commands/mod.rs b/members/nullnet-client/src/commands/mod.rs index 9e32b8f7..4ad95277 100644 --- a/members/nullnet-client/src/commands/mod.rs +++ b/members/nullnet-client/src/commands/mod.rs @@ -25,11 +25,18 @@ pub(crate) async fn setup_br0(rtnetlink_handle: &RtNetLinkHandle) { // delete existing OpenFlow rules OvsCommand::DeleteFlows.execute(); - // use the built-in switching logic - OvsCommand::AddFlow.execute(); - - // add our TAP to the bridge as a trunk port + // add our TAP to the bridge as a trunk port first, so the flow rules + // below can reference it by name OvsCommand::AddTrunkPort.execute(); + + // Route every packet through the TAP (and therefore through + // nullnet-client's encrypting userspace forwarder), instead of letting + // OVS switch same-vlan access ports directly when a tunnel's two + // endpoints happen to be colocated on this same host. Traffic already + // arriving from the trunk (i.e. already decrypted) still gets delivered + // by normal VLAN-aware switching. + OvsCommand::AddTrunkDeliveryFlow.execute(); + OvsCommand::AddAccessRedirectFlow.execute(); } pub(crate) async fn configure_access_port( diff --git a/members/nullnet-client/src/commands/ovs.rs b/members/nullnet-client/src/commands/ovs.rs index 50ad74b0..5ca437fe 100644 --- a/members/nullnet-client/src/commands/ovs.rs +++ b/members/nullnet-client/src/commands/ovs.rs @@ -7,7 +7,16 @@ pub(super) enum OvsCommand<'a> { DeleteBridge, AddBridge, DeleteFlows, - AddFlow, + /// Traffic arriving from the trunk (already decrypted by nullnet-client's + /// userspace forwarder) gets delivered by normal VLAN-aware L2 switching. + AddTrunkDeliveryFlow, + /// Traffic arriving from any access port always goes out the trunk, + /// never directly to another access port. Without this, two access + /// ports for the same vlan_id that happen to live on the same host's + /// bridge (i.e. the tunnel's two endpoints are colocated) would be + /// switched directly by OVS, bypassing the TAP and the encrypting + /// userspace forwarder entirely. + AddAccessRedirectFlow, AddTrunkPort, AddAccessPort(&'a str, u16), } @@ -33,7 +42,9 @@ impl OvsCommand<'_> { | OvsCommand::DeleteBridge | OvsCommand::AddAccessPort(_, _) | OvsCommand::AddTrunkPort => "ovs-vsctl", - OvsCommand::DeleteFlows | OvsCommand::AddFlow => "ovs-ofctl", + OvsCommand::DeleteFlows + | OvsCommand::AddTrunkDeliveryFlow + | OvsCommand::AddAccessRedirectFlow => "ovs-ofctl", } } @@ -45,10 +56,22 @@ impl OvsCommand<'_> { .iter() .map(ToString::to_string) .collect(), - OvsCommand::AddFlow => ["add-flow", "br0", "priority=0,actions=normal"] - .iter() - .map(ToString::to_string) - .collect(), + OvsCommand::AddTrunkDeliveryFlow => [ + "add-flow", + "br0", + &format!("priority=200,in_port={TAP_NAME},actions=normal"), + ] + .iter() + .map(ToString::to_string) + .collect(), + OvsCommand::AddAccessRedirectFlow => [ + "add-flow", + "br0", + &format!("priority=100,actions=output:{TAP_NAME}"), + ] + .iter() + .map(ToString::to_string) + .collect(), OvsCommand::AddTrunkPort => ["add-port", "br0", TAP_NAME] .iter() .map(ToString::to_string) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 50047e85..d7233384 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -59,8 +59,14 @@ fi if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then # Same host: connect bridges with a veth pair instead of a VXLAN tunnel. - # Traffic never leaves the host, so there's nothing to encrypt here — - # no XFRM setup on this branch. + # This traffic never leaves the host, so there's no physical-network + # sniffer to defend against — but it's still worth encrypting for + # defense-in-depth against another, differently-privileged + # container/process on the SAME host that could otherwise read this + # veth's or bridge's plaintext traffic directly. MACsec (802.1AE) wraps + # the veth link itself in AES-256-GCM, keyed with this tunnel's key — + # no IP addressing involved, so it works regardless of what the + # containers on either side are doing. VETH_S="veth-${VXLAN_ID}-s" VETH_C="veth-${VXLAN_ID}-c" # Both ends are created atomically; the losing task's EEXIST is harmless @@ -68,11 +74,33 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then # Attach our end to our bridge if [[ "$BR_NAME" == *_s ]]; then LOCAL_VETH="$VETH_S" + PEER_VETH="$VETH_C" + MACSEC_IF="macsec-${VXLAN_ID}-s" else LOCAL_VETH="$VETH_C" + PEER_VETH="$VETH_S" + MACSEC_IF="macsec-${VXLAN_ID}-c" fi - sudo ip link set "$LOCAL_VETH" master "$BR_NAME" - sudo ip link set "$LOCAL_VETH" mtu $OVERLAY_MTU up + + # The peer's MAC is available immediately: `ip link add ... peer name + # ...` creates both ends atomically in one kernel call, whether this + # invocation won the race above or lost it to the sibling script. + PEER_MAC=$(cat /sys/class/net/$PEER_VETH/address) + KEY_ID=$(printf '%032x' $VXLAN_ID) + + # MACsec adds up to 32 bytes of overhead (SecTAG + ICV for GCM-AES-256). + # Give the underlying veth the extra room — it's a virtual, host-only + # link with no physical MTU constraint — so the macsec interface on + # top of it can still carry a full OVERLAY_MTU-sized frame. + sudo ip link set "$LOCAL_VETH" mtu $((OVERLAY_MTU + 32)) up + + sudo ip link add link "$LOCAL_VETH" "$MACSEC_IF" type macsec cipher gcm-aes-256 port 1 encrypt on 2>/dev/null + sudo ip macsec add "$MACSEC_IF" tx sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" 2>/dev/null + sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" on 2>/dev/null + sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" 2>/dev/null + + sudo ip link set "$MACSEC_IF" master "$BR_NAME" + sudo ip link set "$MACSEC_IF" mtu $OVERLAY_MTU up else # Create the VXLAN tunnel using your physical IP and interface. Each # tunnel gets its own dstport (instead of the IANA-standard 4789) so diff --git a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh index 99e9d25d..a31f2bd8 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh @@ -26,7 +26,13 @@ if [ "$LOCAL_IP" != "$REMOTE_IP" ]; then sudo ip xfrm state delete src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI 2>/dev/null fi -# Remove the VXLAN tunnel or same-host veth pair: +# Remove the VXLAN tunnel or same-host veth pair. Deleting a veth end also +# destroys its peer and cascades to remove any macsec interface stacked on +# either end (the same-host branch of vxlan-setup.sh wraps each end in one), +# but delete both macsec names explicitly too rather than depend solely on +# that cascade. +sudo ip link del macsec-${VXLAN_ID}-s 2>/dev/null +sudo ip link del macsec-${VXLAN_ID}-c 2>/dev/null sudo ip link set vxlan-$NS_NAME down && sudo ip link del vxlan-$NS_NAME sudo ip link set veth-${VXLAN_ID}-s down && sudo ip link del veth-${VXLAN_ID}-s From 4f818ae6614554e42c31a8d611fc28ae301f021b Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Thu, 9 Jul 2026 22:10:50 -0400 Subject: [PATCH 03/13] Fix macsec link-add argument order for same-host VXLAN encryption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit iproute2's macsec option parser is positional: `port` (part of this device's own SCI) must appear before `cipher`, not after. The previous order failed with "macsec: unknown command \"port\"?" every time, silently, because the command's stderr was suppressed to tolerate an unrelated race (the veth-pair creation a few lines earlier). None of the four macsec setup commands actually race against the sibling script invocation — each side only touches its own uniquely-named interface — so their stderr is no longer suppressed either, making a real failure here loud instead of silent next time. Found by live-testing on a real same-host deployment: `ip macsec show` was empty despite the veth pair (and its MTU bump) being created correctly, which pointed straight at the interface-creation line right after it. --- .../nullnet-client/vxlan_scripts/vxlan-setup.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index d7233384..0f8fa92b 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -94,10 +94,19 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then # top of it can still carry a full OVERLAY_MTU-sized frame. sudo ip link set "$LOCAL_VETH" mtu $((OVERLAY_MTU + 32)) up - sudo ip link add link "$LOCAL_VETH" "$MACSEC_IF" type macsec cipher gcm-aes-256 port 1 encrypt on 2>/dev/null - sudo ip macsec add "$MACSEC_IF" tx sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" 2>/dev/null - sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" on 2>/dev/null - sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" 2>/dev/null + # Note the argument order: `port` (part of this device's own SCI) has + # to come before `cipher` — iproute2's macsec option parser is + # positional here, not a free-order keyword scanner, and silently + # rejects `port` if it comes after `cipher` ("unknown command + # \"port\"?"). Unlike the veth-pair creation above, none of these four + # commands race against the sibling script invocation (each side only + # ever touches its own uniquely-named macsec interface), so their + # stderr is deliberately left unsuppressed — a real failure here + # should be loud, not silently swallowed. + sudo ip link add link "$LOCAL_VETH" "$MACSEC_IF" type macsec port 1 cipher gcm-aes-256 encrypt on + sudo ip macsec add "$MACSEC_IF" tx sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" + sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" on + sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" sudo ip link set "$MACSEC_IF" master "$BR_NAME" sudo ip link set "$MACSEC_IF" mtu $OVERLAY_MTU up From a0a46a5884f3bf2845c644d42e0f544097c87a1b Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Thu, 9 Jul 2026 22:56:06 -0400 Subject: [PATCH 04/13] TEMP: add debug logging to VLAN encrypt/decrypt/firewall paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnostic aid for tracking down intermittent "connection reset" on proxied SSH-over-VLAN: every silent-drop point (missing key, decrypt/auth failure, malformed datagram) and every firewall verdict on both send and receive now logs to stderr with [DEBUG] prefix. In particular, receive.rs now logs before crafting a REJECT reply — that's the one code path that sends a real TCP RST back to the peer, so if a REJECT verdict is firing intermittently on legitimate traffic, this will show it directly instead of us guessing from symptoms alone. Intended to be reverted once the root cause is found — not a permanent logging addition. --- members/nullnet-client/src/forward/receive.rs | 30 ++++++++++++++++--- members/nullnet-client/src/forward/send.rs | 17 ++++++++--- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/members/nullnet-client/src/forward/receive.rs b/members/nullnet-client/src/forward/receive.rs index bc53758a..3177f99c 100644 --- a/members/nullnet-client/src/forward/receive.rs +++ b/members/nullnet-client/src/forward/receive.rs @@ -32,27 +32,41 @@ pub async fn receive( // the vlan_id has to be readable before decryption so we know // which tunnel's key to decrypt with let Some((vlan_id, sealed)) = crypto::open_vlan_id(datagram) else { + eprintln!( + "[DEBUG receive] datagram too short to contain a vlan_id (len={}) from {remote_socket} — dropping", + frame.size + ); continue; }; let Some(cipher) = peers.read().await.get_key(vlan_id) else { + eprintln!( + "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: no key in Peers — dropping" + ); continue; }; // decrypt as the packet exits the tunnel; auth failure (wrong // key, corrupted/spoofed datagram) drops it here let Some(pkt_data) = cipher.decrypt(sealed) else { + eprintln!( + "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: decrypt/auth FAILED (len={}) — dropping", + sealed.len() + ); continue; }; - match firewall + let verdict = firewall .read() .await - .resolve_packet(&pkt_data, FirewallDirection::IN) - { + .resolve_packet(&pkt_data, FirewallDirection::IN); + match verdict { FirewallAction::ACCEPT => { // write packet to the kernel device.send(&pkt_data).await.unwrap_or(0); } FirewallAction::REJECT => { + eprintln!( + "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: firewall REJECT — crafting reply" + ); if let Some(reply) = build_termination_message(&pkt_data) && let Some(reply_datagram) = crypto::seal(vlan_id, &cipher, &reply) { @@ -60,9 +74,17 @@ pub async fn receive( .send_to(&reply_datagram, remote_socket) .await .unwrap_or(0); + } else { + eprintln!( + "[DEBUG receive] vlan_id={vlan_id}: REJECT verdict but couldn't build/seal a reply" + ); } } - FirewallAction::DENY => {} + FirewallAction::DENY => { + eprintln!( + "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: firewall DENY — dropping silently" + ); + } } } } diff --git a/members/nullnet-client/src/forward/send.rs b/members/nullnet-client/src/forward/send.rs index 1ebf27ce..b7d2bd3f 100644 --- a/members/nullnet-client/src/forward/send.rs +++ b/members/nullnet-client/src/forward/send.rs @@ -30,23 +30,32 @@ pub async fn send( let Ok((dst_socket, vlan_id)) = get_dst_socket(pkt_data, &peers).await else { continue; }; - match firewall + let verdict = firewall .read() .await - .resolve_packet(pkt_data, FirewallDirection::OUT) - { + .resolve_packet(pkt_data, FirewallDirection::OUT); + match verdict { FirewallAction::ACCEPT => { // encrypt as the packet enters the tunnel: without a key // for this vlan_id (tunnel torn down mid-flight, or // setup never landed) there is nothing safe to send. let Some(cipher) = peers.read().await.get_key(vlan_id) else { + eprintln!( + "[DEBUG send] vlan_id={vlan_id} ACCEPT but no key in Peers — dropping" + ); continue; }; if let Some(datagram) = crypto::seal(vlan_id, &cipher, pkt_data) { socket.send_to(&datagram, dst_socket).await.unwrap_or(0); + } else { + eprintln!("[DEBUG send] vlan_id={vlan_id} seal() failed — dropping"); } } - FirewallAction::DENY | FirewallAction::REJECT => {} + FirewallAction::DENY | FirewallAction::REJECT => { + eprintln!( + "[DEBUG send] vlan_id={vlan_id} OUT verdict={verdict:?} — dropping (no reply sent on OUT)" + ); + } } } } From 37c6334dbb9e9ee4a6e8071ed3c6bcbe7ffae831 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Thu, 9 Jul 2026 23:00:36 -0400 Subject: [PATCH 05/13] Fix VLAN tag loss when redirecting access-port traffic to the trunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same-host defense-in-depth fix replaced OVS's single default flow (priority=0,actions=normal) with a generic "everything from an access port goes to the trunk" rule using a raw `output:` action. Unlike `actions=normal`, `output` does not re-add the 802.1Q tag that access ports only carry internally (as an implicit port association, not part of the packet's own bytes) — so every redirected frame arrived at nullnet-client's TAP already stripped of its VLAN tag, and got dropped by forward/send.rs as malformed ("Packet missing VLAN tag"), silently, for every single packet on every VLAN tunnel that hit this path. Replaced the one generic redirect rule with one precise rule per access port (installed alongside the port itself in configure_access_port, torn down alongside it in remove_vlan), each matching only its own in_port and explicitly pushing the correct 802.1Q tag (push_vlan + mod_vlan_vid) before sending to the trunk. The original priority=0 default-normal rule is restored as a safety fallback for the narrow window before a fresh access port's own rule lands. Found via live testing: intermittent SSH-over-proxy connection resets that traced back to a flood of "Packet missing VLAN tag" errors in nullnet-client's own logs. --- members/nullnet-client/src/commands/mod.rs | 29 +++++++++---- members/nullnet-client/src/commands/ovs.rs | 48 +++++++++++++++++----- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/members/nullnet-client/src/commands/mod.rs b/members/nullnet-client/src/commands/mod.rs index 4ad95277..c67c332a 100644 --- a/members/nullnet-client/src/commands/mod.rs +++ b/members/nullnet-client/src/commands/mod.rs @@ -29,14 +29,16 @@ pub(crate) async fn setup_br0(rtnetlink_handle: &RtNetLinkHandle) { // below can reference it by name OvsCommand::AddTrunkPort.execute(); - // Route every packet through the TAP (and therefore through - // nullnet-client's encrypting userspace forwarder), instead of letting - // OVS switch same-vlan access ports directly when a tunnel's two - // endpoints happen to be colocated on this same host. Traffic already - // arriving from the trunk (i.e. already decrypted) still gets delivered - // by normal VLAN-aware switching. + // Safe fallback (same as OVS's original single default rule) for + // anything not covered by a more specific rule — mainly the brief + // window between an access port being created and its own redirect + // rule (installed in `configure_access_port`) landing. + OvsCommand::AddDefaultFlow.execute(); + + // Traffic arriving from the trunk (i.e. already decrypted by + // nullnet-client's userspace forwarder) is delivered by normal + // VLAN-aware switching. OvsCommand::AddTrunkDeliveryFlow.execute(); - OvsCommand::AddAccessRedirectFlow.execute(); } pub(crate) async fn configure_access_port( @@ -58,9 +60,22 @@ pub(crate) async fn configure_access_port( // add the peer interface to the bridge as an access port OvsCommand::AddAccessPort(&veth_peer_name, vlan_id).execute(); + + // Redirect this port's traffic to the trunk instead of letting OVS + // switch it directly to another local access port — and re-add the + // 802.1Q tag that gets stripped along the way, since the raw `output` + // action used to reach the trunk doesn't do that automatically the way + // `actions=normal` would. + OvsCommand::AddAccessRedirectFlow(&veth_peer_name, vlan_id).execute(); } pub(crate) async fn remove_vlan(rtnetlink_handle: &RtNetLinkHandle, vlan_id: u16) { + // remove this port's redirect flow before the port itself disappears, + // so no stale rule is left behind that could later match a different, + // unrelated port reusing the same OVS port number + let veth_peer_name = format!("veth-{vlan_id}p"); + OvsCommand::DeleteAccessRedirectFlow(&veth_peer_name).execute(); + // delete the veth pair rtnetlink_handle .execute(NetLinkCommand::DeleteVeth(vlan_id)) diff --git a/members/nullnet-client/src/commands/ovs.rs b/members/nullnet-client/src/commands/ovs.rs index 5ca437fe..51d2c69f 100644 --- a/members/nullnet-client/src/commands/ovs.rs +++ b/members/nullnet-client/src/commands/ovs.rs @@ -7,16 +7,30 @@ pub(super) enum OvsCommand<'a> { DeleteBridge, AddBridge, DeleteFlows, + /// Fallback for anything not covered by a more specific rule below — + /// same behavior as OVS's original single default flow. Mainly covers + /// the brief startup window between an access port being created and + /// its own redirect flow (below) landing. + AddDefaultFlow, /// Traffic arriving from the trunk (already decrypted by nullnet-client's /// userspace forwarder) gets delivered by normal VLAN-aware L2 switching. AddTrunkDeliveryFlow, - /// Traffic arriving from any access port always goes out the trunk, - /// never directly to another access port. Without this, two access - /// ports for the same vlan_id that happen to live on the same host's - /// bridge (i.e. the tunnel's two endpoints are colocated) would be - /// switched directly by OVS, bypassing the TAP and the encrypting - /// userspace forwarder entirely. - AddAccessRedirectFlow, + /// One rule per access port, installed alongside it: redirect this + /// port's traffic to the trunk instead of letting OVS switch it + /// directly to another local access port (which would bypass the TAP + /// and the encrypting userspace forwarder entirely when a tunnel's two + /// endpoints happen to be colocated on this host). `output:` is a + /// raw action — unlike `actions=normal`, it does *not* re-add the + /// 802.1Q tag that access ports carry only internally, so this + /// explicitly pushes the tag back on first: without that, packets would + /// arrive at nullnet-client's TAP already stripped of their VLAN tag + /// and get silently dropped as malformed. + AddAccessRedirectFlow(&'a str, u16), + /// Removes exactly the rule `AddAccessRedirectFlow` installed for this + /// port, so a torn-down tunnel doesn't leave a stale flow entry that + /// could wrongly match a future, unrelated port reusing the same + /// OVS port number. + DeleteAccessRedirectFlow(&'a str), AddTrunkPort, AddAccessPort(&'a str, u16), } @@ -43,8 +57,10 @@ impl OvsCommand<'_> { | OvsCommand::AddAccessPort(_, _) | OvsCommand::AddTrunkPort => "ovs-vsctl", OvsCommand::DeleteFlows + | OvsCommand::AddDefaultFlow | OvsCommand::AddTrunkDeliveryFlow - | OvsCommand::AddAccessRedirectFlow => "ovs-ofctl", + | OvsCommand::AddAccessRedirectFlow(_, _) + | OvsCommand::DeleteAccessRedirectFlow(_) => "ovs-ofctl", } } @@ -56,6 +72,10 @@ impl OvsCommand<'_> { .iter() .map(ToString::to_string) .collect(), + OvsCommand::AddDefaultFlow => ["add-flow", "br0", "priority=0,actions=normal"] + .iter() + .map(ToString::to_string) + .collect(), OvsCommand::AddTrunkDeliveryFlow => [ "add-flow", "br0", @@ -64,14 +84,22 @@ impl OvsCommand<'_> { .iter() .map(ToString::to_string) .collect(), - OvsCommand::AddAccessRedirectFlow => [ + OvsCommand::AddAccessRedirectFlow(dev, vlan) => [ "add-flow", "br0", - &format!("priority=100,actions=output:{TAP_NAME}"), + &format!( + "priority=150,in_port={dev},actions=push_vlan:0x8100,mod_vlan_vid:{vlan},output:{TAP_NAME}" + ), ] .iter() .map(ToString::to_string) .collect(), + OvsCommand::DeleteAccessRedirectFlow(dev) => { + ["del-flows", "br0", &format!("in_port={dev}")] + .iter() + .map(ToString::to_string) + .collect() + } OvsCommand::AddTrunkPort => ["add-port", "br0", TAP_NAME] .iter() .map(ToString::to_string) From bf2716be43d118cddf40d320b18f4dfc87c2b831 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Thu, 9 Jul 2026 23:21:31 -0400 Subject: [PATCH 06/13] TEMP: log the actual net-layer headers on "unsupported protocol" drops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same diagnostic purpose as the earlier debug commit — need to see what this frame actually was (IPv6 NDP/multicast noise vs. something that should have been carried) rather than just knowing the match fell through to the catch-all arm. --- members/nullnet-client/src/forward/send.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/members/nullnet-client/src/forward/send.rs b/members/nullnet-client/src/forward/send.rs index b7d2bd3f..d8b9b426 100644 --- a/members/nullnet-client/src/forward/send.rs +++ b/members/nullnet-client/src/forward/send.rs @@ -79,7 +79,13 @@ async fn get_dst_socket( .handle_err(location!()), _ => Err("ARP packet with non-IPv4 protocol address type").handle_err(location!()), }, - _ => Err("Unsupported network layer protocol").handle_err(location!()), + _ => { + eprintln!( + "[DEBUG send] vlan_id={vlan_id} unsupported network layer, net={:?}", + headers.net + ); + Err("Unsupported network layer protocol").handle_err(location!()) + } }?; let dest_ip = Ipv4Addr::from(dest_ip_slice); let veth_key = VethKey::new(dest_ip, vlan_id); From e0a0de88e694dbc2d5ece6d6102f020d9f95a6ce Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Sun, 12 Jul 2026 18:44:48 -0400 Subject: [PATCH 07/13] Fix xfrm policy argument order for cross-host VXLAN encryption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same class of bug as the earlier macsec argument-order fix: `ip xfrm policy add/delete` requires the full selector (src/dst/proto/dport) to appear as one contiguous block, with `dir` only afterward — confirmed against `ip xfrm policy help`'s own grammar (`SELECTOR dir DIR ...`). The previous order put `dir out`/`dir in` in the middle of the selector, between dst and proto, which the parser misreads as two separate `proto` values ("duplicate \"unknown\": \"proto\" is the second value") and rejects outright — so no policy was ever installed on either side of a genuinely cross-host VXLAN tunnel, and traffic between the two hosts had no route (the two sides' overlay subnets are meant to look like one flat L2 domain over the VXLAN link, which never got its required IPsec policy in place). Found via live cross-host testing: an sshd service hosted on a second, non-colocated node was unreachable ("No route to host") because its side of the tunnel's bridge/VXLAN interface never got created at all — traced back to this exact error in the client's own log output. --- members/nullnet-client/vxlan_scripts/vxlan-setup.sh | 11 +++++++++-- .../nullnet-client/vxlan_scripts/vxlan-teardown.sh | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 0f8fa92b..ab649aab 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -135,15 +135,22 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then SPI=$(printf '0x%08x' "$VXLAN_ID") # Outbound: this host -> remote. + # Note the argument order: the selector (src/dst/proto/dport) has to + # stay contiguous, with `dir` appearing only after it's complete — + # same lesson as the macsec argument-order bug. Splitting the + # selector by putting `dir` in the middle of it confuses the parser + # into thinking `proto` was given twice ("duplicate \"unknown\": + # \"proto\" is the second value"), and the policy silently never + # gets installed. sudo ip xfrm state add src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI \ mode transport aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 - sudo ip xfrm policy add src $LOCAL_IP dst $REMOTE_IP dir out proto udp dport $DSTPORT \ + sudo ip xfrm policy add src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out \ tmpl src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI mode transport # Inbound: remote -> this host. sudo ip xfrm state add src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI \ mode transport aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 - sudo ip xfrm policy add src $REMOTE_IP dst $LOCAL_IP dir in proto udp dport $DSTPORT \ + sudo ip xfrm policy add src $REMOTE_IP dst $LOCAL_IP proto udp dport $DSTPORT dir in \ tmpl src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI mode transport fi diff --git a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh index a31f2bd8..c61f8cce 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh @@ -20,9 +20,11 @@ DOCKER_CONTAINER=$7 # same-host branch of vxlan-setup.sh never creates one). if [ "$LOCAL_IP" != "$REMOTE_IP" ]; then SPI=$(printf '0x%08x' "$VXLAN_ID") - sudo ip xfrm policy delete src $LOCAL_IP dst $REMOTE_IP dir out proto udp dport $DSTPORT 2>/dev/null + # Same argument-order requirement as vxlan-setup.sh: selector fields + # (src/dst/proto/dport) must stay contiguous, with `dir` only after. + sudo ip xfrm policy delete src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out 2>/dev/null sudo ip xfrm state delete src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI 2>/dev/null - sudo ip xfrm policy delete src $REMOTE_IP dst $LOCAL_IP dir in proto udp dport $DSTPORT 2>/dev/null + sudo ip xfrm policy delete src $REMOTE_IP dst $LOCAL_IP proto udp dport $DSTPORT dir in 2>/dev/null sudo ip xfrm state delete src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI 2>/dev/null fi From 1e13ad9ab75725764080802865832429fd99741e Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Sun, 12 Jul 2026 18:57:31 -0400 Subject: [PATCH 08/13] Fix xfrm state argument order for cross-host VXLAN encryption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same class of bug as the policy argument-order fix, one command over: `ip xfrm state add` requires ID [ALGO-LIST] [mode MODE] ... per its own help text — the algorithm clause (`aead ...`) has to come before `mode`, not after. The previous order (`mode transport aead ...`) is backwards and fails outright with a bare "RTNETLINK answers: Invalid argument", so no Security Association was ever installed on either side of a cross-host VXLAN tunnel — encryption never had a chance to run at all. --- .../vxlan_scripts/vxlan-setup.sh | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index ab649aab..727c4be5 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -135,21 +135,27 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then SPI=$(printf '0x%08x' "$VXLAN_ID") # Outbound: this host -> remote. - # Note the argument order: the selector (src/dst/proto/dport) has to - # stay contiguous, with `dir` appearing only after it's complete — - # same lesson as the macsec argument-order bug. Splitting the - # selector by putting `dir` in the middle of it confuses the parser - # into thinking `proto` was given twice ("duplicate \"unknown\": - # \"proto\" is the second value"), and the policy silently never - # gets installed. + # Note the argument order in both commands below — same lesson as + # the macsec argument-order bug, `ip xfrm` is positional, not a + # free-order keyword scanner: + # - `state add`: the ALGO-LIST (`aead ...`) must come before + # `mode`, not after — "ID [ALGO-LIST] [mode MODE] ..." per + # `ip xfrm state help`. Reversed, it fails with a bare + # "RTNETLINK answers: Invalid argument". + # - `policy add`: the selector (src/dst/proto/dport) must stay + # contiguous, with `dir` only appearing after it's complete — + # "SELECTOR dir DIR ..." per `ip xfrm policy help`. Splitting it + # by putting `dir` in the middle confuses the parser into + # thinking `proto` was given twice ("duplicate \"unknown\": + # \"proto\" is the second value"). sudo ip xfrm state add src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI \ - mode transport aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 + aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 mode transport sudo ip xfrm policy add src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out \ tmpl src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI mode transport # Inbound: remote -> this host. sudo ip xfrm state add src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI \ - mode transport aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 + aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 mode transport sudo ip xfrm policy add src $REMOTE_IP dst $LOCAL_IP proto udp dport $DSTPORT dir in \ tmpl src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI mode transport fi From dba3542000215ce892de10986ae30de9486d84e4 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Sun, 12 Jul 2026 19:17:51 -0400 Subject: [PATCH 09/13] TEMP: enable bash -x tracing in vxlan-setup.sh to find remaining xfrm bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state-vs-policy argument-order fixes so far were based on reading `ip xfrm state help` / `ip xfrm policy help`'s grammar summaries, not confirmed against the actual parser behavior for every command — and the "RTNETLINK answers: Invalid argument" error is still occurring after both fixes, meaning at least one of those diagnoses was incomplete or wrong. Rather than keep guessing from documentation, trace every command with its real substituted values so the next failure shows the exact command line next to its exact error. Prints AEAD key material to stderr while enabled — treat logs as sensitive until this is reverted. --- members/nullnet-client/vxlan_scripts/vxlan-setup.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 727c4be5..2d752b91 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -1,5 +1,14 @@ #!/bin/bash +# TEMP DEBUG: print every command with its actual substituted values to +# stderr right before running it, so a failure shows the exact real command +# next to its exact error instead of us guessing from documentation. +# NOTE: this will print the AEAD key material (derived from the tunnel's +# AES-256 key) into whatever captures nullnet-client's stderr — treat those +# logs as sensitive while this is enabled, and remove this line once the +# xfrm argument-order issue is found. +set -x + # Read CLI arguments: if [ "$#" -lt 9 ] || [ "$#" -gt 10 ]; then echo "Usage: $0 [docker_container]" From c7628f9c0560666537bafe0af0a24a799e1a7c37 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Sun, 12 Jul 2026 19:54:07 -0400 Subject: [PATCH 10/13] Fix xfrm SPI colliding with IANA-reserved range for low vxlan_ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VXLAN IDs start at 101 (net_id_pool.rs's MIN_NET_ID), and the SPI was derived directly from the raw vxlan_id — landing squarely in SPI values 1-255, which RFC 4301 reserves for IANA and which the kernel's XFRM code rejects outright with a bare "RTNETLINK answers: Invalid argument". This was the actual cause of the state-add failures, not an argument-order issue (confirmed by testing with two genuinely real, non-loopback addresses and still hitting the identical error). Offset the SPI by a fixed +1000 so it's always comfortably clear of the reserved range, matching between vxlan-setup.sh (install) and vxlan-teardown.sh (delete, which must derive the identical SPI to remove the right SA). --- members/nullnet-client/vxlan_scripts/vxlan-setup.sh | 6 +++++- members/nullnet-client/vxlan_scripts/vxlan-teardown.sh | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 2d752b91..b78b4b0a 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -141,7 +141,11 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then # only reproducible from the shared secret both sides already have. SALT_HEX=$(printf '%s' "$KEY_HEX" | sha256sum | cut -c1-8) AEAD_KEY_HEX="${KEY_HEX}${SALT_HEX}" - SPI=$(printf '0x%08x' "$VXLAN_ID") + # SPI values 1-255 are IANA-reserved (RFC 4301) and the kernel's XFRM + # code rejects them outright ("Invalid argument"). vxlan_id starts at + # 101 (see net_id_pool.rs), which falls straight into that reserved + # range — offset it well clear of 255 rather than using the raw ID. + SPI=$(printf '0x%08x' $((VXLAN_ID + 1000))) # Outbound: this host -> remote. # Note the argument order in both commands below — same lesson as diff --git a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh index c61f8cce..f64ae11f 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh @@ -19,7 +19,9 @@ DOCKER_CONTAINER=$7 # Remove this tunnel's XFRM state + policy pair, if any was installed (the # same-host branch of vxlan-setup.sh never creates one). if [ "$LOCAL_IP" != "$REMOTE_IP" ]; then - SPI=$(printf '0x%08x' "$VXLAN_ID") + # Must match the same offset vxlan-setup.sh uses, to delete the actual + # installed SPI rather than the raw (and IANA-reserved) vxlan_id. + SPI=$(printf '0x%08x' $((VXLAN_ID + 1000))) # Same argument-order requirement as vxlan-setup.sh: selector fields # (src/dst/proto/dport) must stay contiguous, with `dir` only after. sudo ip xfrm policy delete src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out 2>/dev/null From 7aa13034c01074359a0d26032f35d028578281bd Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Sun, 12 Jul 2026 20:33:58 -0400 Subject: [PATCH 11/13] Fix xfrm state ALGO-KEYMAT missing required 0x prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actual root cause behind every "RTNETLINK answers: Invalid argument" we chased across the SPI-range and argument-order fixes: `ip xfrm state add`'s key material (ALGO-KEYMAT) must be prefixed with "0x" — a bare hex string of the exact same value and length is silently rejected by the kernel. Confirmed by extensive live A/B testing on a real target host: identical algorithm, key length, and argument order succeeded the moment "0x" was added and failed identically without it, for both classic enc+auth (cbc(aes)+hmac(sha1)) and this script's actual aead (rfc4106(gcm(aes))) construction. The SPI-reserved-range fix (c7628f9) and the state/policy argument-order fixes (e0a0de8, 1e13ad9) were real, independently-necessary corrections found along the way — this was the last remaining piece keeping cross-host VXLAN encryption from ever actually installing an SA. --- members/nullnet-client/vxlan_scripts/vxlan-setup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index b78b4b0a..2c87b4c9 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -140,7 +140,10 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then # ends, from that same key — it doesn't need to be secret on its own, # only reproducible from the shared secret both sides already have. SALT_HEX=$(printf '%s' "$KEY_HEX" | sha256sum | cut -c1-8) - AEAD_KEY_HEX="${KEY_HEX}${SALT_HEX}" + # `ip xfrm state add`'s ALGO-KEYMAT requires a "0x" prefix — a bare hex + # string is rejected outright with a bare "RTNETLINK answers: Invalid + # argument", confirmed by extensive live testing (see commit history). + AEAD_KEY_HEX="0x${KEY_HEX}${SALT_HEX}" # SPI values 1-255 are IANA-reserved (RFC 4301) and the kernel's XFRM # code rejects them outright ("Invalid argument"). vxlan_id starts at # 101 (see net_id_pool.rs), which falls straight into that reserved From 35c2d1e5c0152fad7085abea4ddd322d72c49a92 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Sun, 12 Jul 2026 20:34:13 -0400 Subject: [PATCH 12/13] Revert "TEMP: enable bash -x tracing in vxlan-setup.sh to find remaining xfrm bug" This reverts commit dba3542000215ce892de10986ae30de9486d84e4. --- members/nullnet-client/vxlan_scripts/vxlan-setup.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 2c87b4c9..eb849330 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -1,14 +1,5 @@ #!/bin/bash -# TEMP DEBUG: print every command with its actual substituted values to -# stderr right before running it, so a failure shows the exact real command -# next to its exact error instead of us guessing from documentation. -# NOTE: this will print the AEAD key material (derived from the tunnel's -# AES-256 key) into whatever captures nullnet-client's stderr — treat those -# logs as sensitive while this is enabled, and remove this line once the -# xfrm argument-order issue is found. -set -x - # Read CLI arguments: if [ "$#" -lt 9 ] || [ "$#" -gt 10 ]; then echo "Usage: $0 [docker_container]" From 9d194c520a530aa2067779e25e3305acc80124d4 Mon Sep 17 00:00:00 2001 From: Anton Liashkevich Date: Sun, 12 Jul 2026 20:56:21 -0400 Subject: [PATCH 13/13] Remove temporary debug logging from VLAN forward/send and receive The VLAN-tag-loss bug and the unsupported-network-layer noise these were added to diagnose (a0a46a5, bf2716b) are both confirmed fixed and verified working via live testing. Restores forward/send.rs and forward/receive.rs to their clean, non-instrumented state. --- members/nullnet-client/src/forward/receive.rs | 30 +++---------------- members/nullnet-client/src/forward/send.rs | 25 ++++------------ 2 files changed, 9 insertions(+), 46 deletions(-) diff --git a/members/nullnet-client/src/forward/receive.rs b/members/nullnet-client/src/forward/receive.rs index 3177f99c..bc53758a 100644 --- a/members/nullnet-client/src/forward/receive.rs +++ b/members/nullnet-client/src/forward/receive.rs @@ -32,41 +32,27 @@ pub async fn receive( // the vlan_id has to be readable before decryption so we know // which tunnel's key to decrypt with let Some((vlan_id, sealed)) = crypto::open_vlan_id(datagram) else { - eprintln!( - "[DEBUG receive] datagram too short to contain a vlan_id (len={}) from {remote_socket} — dropping", - frame.size - ); continue; }; let Some(cipher) = peers.read().await.get_key(vlan_id) else { - eprintln!( - "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: no key in Peers — dropping" - ); continue; }; // decrypt as the packet exits the tunnel; auth failure (wrong // key, corrupted/spoofed datagram) drops it here let Some(pkt_data) = cipher.decrypt(sealed) else { - eprintln!( - "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: decrypt/auth FAILED (len={}) — dropping", - sealed.len() - ); continue; }; - let verdict = firewall + match firewall .read() .await - .resolve_packet(&pkt_data, FirewallDirection::IN); - match verdict { + .resolve_packet(&pkt_data, FirewallDirection::IN) + { FirewallAction::ACCEPT => { // write packet to the kernel device.send(&pkt_data).await.unwrap_or(0); } FirewallAction::REJECT => { - eprintln!( - "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: firewall REJECT — crafting reply" - ); if let Some(reply) = build_termination_message(&pkt_data) && let Some(reply_datagram) = crypto::seal(vlan_id, &cipher, &reply) { @@ -74,17 +60,9 @@ pub async fn receive( .send_to(&reply_datagram, remote_socket) .await .unwrap_or(0); - } else { - eprintln!( - "[DEBUG receive] vlan_id={vlan_id}: REJECT verdict but couldn't build/seal a reply" - ); } } - FirewallAction::DENY => { - eprintln!( - "[DEBUG receive] vlan_id={vlan_id} from {remote_socket}: firewall DENY — dropping silently" - ); - } + FirewallAction::DENY => {} } } } diff --git a/members/nullnet-client/src/forward/send.rs b/members/nullnet-client/src/forward/send.rs index d8b9b426..1ebf27ce 100644 --- a/members/nullnet-client/src/forward/send.rs +++ b/members/nullnet-client/src/forward/send.rs @@ -30,32 +30,23 @@ pub async fn send( let Ok((dst_socket, vlan_id)) = get_dst_socket(pkt_data, &peers).await else { continue; }; - let verdict = firewall + match firewall .read() .await - .resolve_packet(pkt_data, FirewallDirection::OUT); - match verdict { + .resolve_packet(pkt_data, FirewallDirection::OUT) + { FirewallAction::ACCEPT => { // encrypt as the packet enters the tunnel: without a key // for this vlan_id (tunnel torn down mid-flight, or // setup never landed) there is nothing safe to send. let Some(cipher) = peers.read().await.get_key(vlan_id) else { - eprintln!( - "[DEBUG send] vlan_id={vlan_id} ACCEPT but no key in Peers — dropping" - ); continue; }; if let Some(datagram) = crypto::seal(vlan_id, &cipher, pkt_data) { socket.send_to(&datagram, dst_socket).await.unwrap_or(0); - } else { - eprintln!("[DEBUG send] vlan_id={vlan_id} seal() failed — dropping"); } } - FirewallAction::DENY | FirewallAction::REJECT => { - eprintln!( - "[DEBUG send] vlan_id={vlan_id} OUT verdict={verdict:?} — dropping (no reply sent on OUT)" - ); - } + FirewallAction::DENY | FirewallAction::REJECT => {} } } } @@ -79,13 +70,7 @@ async fn get_dst_socket( .handle_err(location!()), _ => Err("ARP packet with non-IPv4 protocol address type").handle_err(location!()), }, - _ => { - eprintln!( - "[DEBUG send] vlan_id={vlan_id} unsupported network layer, net={:?}", - headers.net - ); - Err("Unsupported network layer protocol").handle_err(location!()) - } + _ => Err("Unsupported network layer protocol").handle_err(location!()), }?; let dest_ip = Ipv4Addr::from(dest_ip_slice); let veth_key = VethKey::new(dest_ip, vlan_id);