diff --git a/Cargo.lock b/Cargo.lock index e04dfcb2..28a34a47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2769,6 +2769,7 @@ dependencies = [ name = "nullnet-client" version = "0.1.0" dependencies = [ + "aes-gcm", "chrono", "clap", "etherparse", diff --git a/members/nullnet-client/Cargo.toml b/members/nullnet-client/Cargo.toml index 79affbb6..0c6fb178 100644 --- a/members/nullnet-client/Cargo.toml +++ b/members/nullnet-client/Cargo.toml @@ -23,4 +23,5 @@ futures = "0.3.32" network-interface = "2.0.5" gag.workspace = true chrono.workspace = true -nfq = "0.2" \ No newline at end of file +nfq = "0.2" +aes-gcm = "0.10" \ No newline at end of file diff --git a/members/nullnet-client/src/commands/mod.rs b/members/nullnet-client/src/commands/mod.rs index 9e32b8f7..c67c332a 100644 --- a/members/nullnet-client/src/commands/mod.rs +++ b/members/nullnet-client/src/commands/mod.rs @@ -25,11 +25,20 @@ pub(crate) async fn setup_br0(rtnetlink_handle: &RtNetLinkHandle) { // delete existing OpenFlow rules OvsCommand::DeleteFlows.execute(); - // use the built-in switching logic - OvsCommand::AddFlow.execute(); - - // add our TAP to the bridge as a trunk port + // add our TAP to the bridge as a trunk port first, so the flow rules + // below can reference it by name OvsCommand::AddTrunkPort.execute(); + + // Safe fallback (same as OVS's original single default rule) for + // anything not covered by a more specific rule — mainly the brief + // window between an access port being created and its own redirect + // rule (installed in `configure_access_port`) landing. + OvsCommand::AddDefaultFlow.execute(); + + // Traffic arriving from the trunk (i.e. already decrypted by + // nullnet-client's userspace forwarder) is delivered by normal + // VLAN-aware switching. + OvsCommand::AddTrunkDeliveryFlow.execute(); } pub(crate) async fn configure_access_port( @@ -51,9 +60,22 @@ pub(crate) async fn configure_access_port( // add the peer interface to the bridge as an access port OvsCommand::AddAccessPort(&veth_peer_name, vlan_id).execute(); + + // Redirect this port's traffic to the trunk instead of letting OVS + // switch it directly to another local access port — and re-add the + // 802.1Q tag that gets stripped along the way, since the raw `output` + // action used to reach the trunk doesn't do that automatically the way + // `actions=normal` would. + OvsCommand::AddAccessRedirectFlow(&veth_peer_name, vlan_id).execute(); } pub(crate) async fn remove_vlan(rtnetlink_handle: &RtNetLinkHandle, vlan_id: u16) { + // remove this port's redirect flow before the port itself disappears, + // so no stale rule is left behind that could later match a different, + // unrelated port reusing the same OVS port number + let veth_peer_name = format!("veth-{vlan_id}p"); + OvsCommand::DeleteAccessRedirectFlow(&veth_peer_name).execute(); + // delete the veth pair rtnetlink_handle .execute(NetLinkCommand::DeleteVeth(vlan_id)) diff --git a/members/nullnet-client/src/commands/ovs.rs b/members/nullnet-client/src/commands/ovs.rs index 50ad74b0..51d2c69f 100644 --- a/members/nullnet-client/src/commands/ovs.rs +++ b/members/nullnet-client/src/commands/ovs.rs @@ -7,7 +7,30 @@ pub(super) enum OvsCommand<'a> { DeleteBridge, AddBridge, DeleteFlows, - AddFlow, + /// Fallback for anything not covered by a more specific rule below — + /// same behavior as OVS's original single default flow. Mainly covers + /// the brief startup window between an access port being created and + /// its own redirect flow (below) landing. + AddDefaultFlow, + /// Traffic arriving from the trunk (already decrypted by nullnet-client's + /// userspace forwarder) gets delivered by normal VLAN-aware L2 switching. + AddTrunkDeliveryFlow, + /// One rule per access port, installed alongside it: redirect this + /// port's traffic to the trunk instead of letting OVS switch it + /// directly to another local access port (which would bypass the TAP + /// and the encrypting userspace forwarder entirely when a tunnel's two + /// endpoints happen to be colocated on this host). `output:` is a + /// raw action — unlike `actions=normal`, it does *not* re-add the + /// 802.1Q tag that access ports carry only internally, so this + /// explicitly pushes the tag back on first: without that, packets would + /// arrive at nullnet-client's TAP already stripped of their VLAN tag + /// and get silently dropped as malformed. + AddAccessRedirectFlow(&'a str, u16), + /// Removes exactly the rule `AddAccessRedirectFlow` installed for this + /// port, so a torn-down tunnel doesn't leave a stale flow entry that + /// could wrongly match a future, unrelated port reusing the same + /// OVS port number. + DeleteAccessRedirectFlow(&'a str), AddTrunkPort, AddAccessPort(&'a str, u16), } @@ -33,7 +56,11 @@ impl OvsCommand<'_> { | OvsCommand::DeleteBridge | OvsCommand::AddAccessPort(_, _) | OvsCommand::AddTrunkPort => "ovs-vsctl", - OvsCommand::DeleteFlows | OvsCommand::AddFlow => "ovs-ofctl", + OvsCommand::DeleteFlows + | OvsCommand::AddDefaultFlow + | OvsCommand::AddTrunkDeliveryFlow + | OvsCommand::AddAccessRedirectFlow(_, _) + | OvsCommand::DeleteAccessRedirectFlow(_) => "ovs-ofctl", } } @@ -45,10 +72,34 @@ impl OvsCommand<'_> { .iter() .map(ToString::to_string) .collect(), - OvsCommand::AddFlow => ["add-flow", "br0", "priority=0,actions=normal"] + OvsCommand::AddDefaultFlow => ["add-flow", "br0", "priority=0,actions=normal"] .iter() .map(ToString::to_string) .collect(), + OvsCommand::AddTrunkDeliveryFlow => [ + "add-flow", + "br0", + &format!("priority=200,in_port={TAP_NAME},actions=normal"), + ] + .iter() + .map(ToString::to_string) + .collect(), + OvsCommand::AddAccessRedirectFlow(dev, vlan) => [ + "add-flow", + "br0", + &format!( + "priority=150,in_port={dev},actions=push_vlan:0x8100,mod_vlan_vid:{vlan},output:{TAP_NAME}" + ), + ] + .iter() + .map(ToString::to_string) + .collect(), + OvsCommand::DeleteAccessRedirectFlow(dev) => { + ["del-flows", "br0", &format!("in_port={dev}")] + .iter() + .map(ToString::to_string) + .collect() + } OvsCommand::AddTrunkPort => ["add-port", "br0", TAP_NAME] .iter() .map(ToString::to_string) diff --git a/members/nullnet-client/src/control_channel.rs b/members/nullnet-client/src/control_channel.rs index ccfa0588..ba01b631 100644 --- a/members/nullnet-client/src/control_channel.rs +++ b/members/nullnet-client/src/control_channel.rs @@ -162,6 +162,24 @@ async fn handle_vlan_setup( }), ); })?; + // Fail before touching the network if the key is malformed — running + // this tunnel without a valid key would mean forwarding traffic in the + // clear instead of encrypted. + let encryption_key: [u8; 32] = message + .encryption_key + .try_into() + .map_err(|_| "VLAN setup message carried a malformed encryption key") + .handle_err(location!()) + .inspect_err(|e| { + fire_event( + &grpc, + AgentEventKind::VlanSetupFailed(AgentVlanSetupFailed { + vlan_id: message.vlan_id, + local_veth: local_veth.to_string(), + error_reason: e.to_str().to_string(), + }), + ); + })?; // setup VLAN on this machine let init_t = std::time::Instant::now(); @@ -176,11 +194,12 @@ async fn handle_vlan_setup( init_t.elapsed().as_millis() ); - // register peer - peers - .write() - .await - .insert(VethKey::new(remote_veth, vlan_id), remote_ip); + // register peer + this tunnel's encryption key + { + let mut peers = peers.write().await; + peers.insert(VethKey::new(remote_veth, vlan_id), remote_ip); + peers.insert_key(vlan_id, &encryption_key); + } // add host mapping if needed if let Some(host_mapping) = &message.host_mapping { @@ -288,6 +307,25 @@ async fn handle_vxlan_setup( .remote_ip .parse::() .handle_err(location!())?; + // Fail before touching the network if the key is malformed — running + // this tunnel without a valid key would mean forwarding traffic in the + // clear instead of encrypted. + let encryption_key: [u8; 32] = message + .encryption_key + .try_into() + .map_err(|_| "VXLAN setup message carried a malformed encryption key") + .handle_err(location!()) + .inspect_err(|e| { + fire_event( + &grpc, + AgentEventKind::VxlanSetupFailed(AgentVxlanSetupFailed { + vxlan_id, + ns_name: ns_name.clone(), + error_code: -1, + }), + ); + eprintln!("[vxlan_setup] {}", e.to_str()); + })?; // setup VXLAN on this machine (optionally attaching a Docker container) let init_t = std::time::Instant::now(); @@ -298,7 +336,9 @@ async fn handle_vxlan_setup( .arg(br_name) .arg(br_net.to_string()) .arg(local_ip.to_string()) - .arg(remote_ip.to_string()); + .arg(remote_ip.to_string()) + .arg(hex_encode(&encryption_key)) + .arg(message.dstport.to_string()); if let Some(container) = &message.docker_container { cmd.arg(container); } @@ -459,7 +499,12 @@ fn handle_vxlan_teardown( let br_name = message.br_name; let mut cmd = std::process::Command::new("./vxlan_scripts/vxlan-teardown.sh"); - cmd.arg(vxlan_id.to_string()).arg(&ns_name).arg(&br_name); + cmd.arg(vxlan_id.to_string()) + .arg(&ns_name) + .arg(&br_name) + .arg(&message.local_ip) + .arg(&message.remote_ip) + .arg(message.dstport.to_string()); if let Some(container) = &message.docker_container { cmd.arg(container); } @@ -710,3 +755,9 @@ fn remove_hosts_entry(content: &str, name: &str) -> String { .collect(); lines.join("\n") + "\n" } + +/// Lowercase hex encoding, used to pass the tunnel's AES key to +/// `vxlan-setup.sh`/`vxlan-teardown.sh` as a shell argument. +fn hex_encode(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/members/nullnet-client/src/craft/reject_payloads.rs b/members/nullnet-client/src/craft/reject_payloads.rs index 52ddaf05..b2795951 100644 --- a/members/nullnet-client/src/craft/reject_payloads.rs +++ b/members/nullnet-client/src/craft/reject_payloads.rs @@ -3,66 +3,45 @@ use etherparse::{ Icmpv4Header, Icmpv4Type, IpFragOffset, IpNumber, LaxPacketHeaders, LinkExtHeader, LinkHeader, NetHeaders, TcpOptions, TransportHeader, }; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::net::UdpSocket; -/// Sends a proper message to gracefully acknowledge a peer that a packet was rejected, +/// Builds a proper message to gracefully acknowledge a peer that a packet was rejected, /// based on the observed protocol: /// - in case of TCP, a packet with RST and ACK flag is sent /// - in case of UDP, an ICMP port unreachable message is sent /// - in case of other protocols, an ICMP host unreachable message is sent -pub async fn send_termination_message( - packet: &[u8], - socket: &Arc, - remote_socket: SocketAddr, -) { - let Ok(headers) = LaxPacketHeaders::from_ethernet(packet) else { - return; - }; +/// +/// Returns the plaintext Ethernet frame to send; the caller (`forward/receive.rs`) +/// is responsible for encrypting and framing it before it hits the wire, same as +/// any other outbound frame on the VLAN forwarder. +pub fn build_termination_message(packet: &[u8]) -> Option> { + let headers = LaxPacketHeaders::from_ethernet(packet).ok()?; let Some(NetHeaders::Ipv4(ip_header, _)) = &headers.net else { - return; + return None; }; let IpNumber(proto) = ip_header.protocol; match proto { - 6 => Box::pin(send_tcp_rst(headers, socket, remote_socket)).await, + 6 => send_tcp_rst(headers), 17 => { // port unreachable let icmp_type = Icmpv4Type::DestinationUnreachable(DestUnreachableHeader::Port); - Box::pin(send_destination_unreachable( - packet, - headers, - socket, - icmp_type, - remote_socket, - )) - .await; + send_destination_unreachable(packet, headers, icmp_type) } _ => { // host unreachable let icmp_type = Icmpv4Type::DestinationUnreachable(DestUnreachableHeader::Host); - Box::pin(send_destination_unreachable( - packet, - headers, - socket, - icmp_type, - remote_socket, - )) - .await; + send_destination_unreachable(packet, headers, icmp_type) } } } -async fn send_destination_unreachable( +fn send_destination_unreachable( packet: &[u8], headers: LaxPacketHeaders<'_>, - socket: &Arc, icmp_type: Icmpv4Type, - remote_socket: SocketAddr, -) { +) -> Option> { let Some(LinkHeader::Ethernet2(mut ethernet_header)) = headers.link else { - return; + return None; }; std::mem::swap( &mut ethernet_header.source, @@ -80,7 +59,7 @@ async fn send_destination_unreachable( .collect(); let Some(NetHeaders::Ipv4(mut ip_header, _)) = headers.net else { - return; + return None; }; let original_ip_header_bytes = ip_header.to_bytes(); let size_up_to_ip_header = @@ -112,19 +91,12 @@ async fn send_destination_unreachable( &icmp_payload[..], ].concat(); - socket - .send_to(&pkt_response, remote_socket) - .await - .unwrap_or(0); + Some(pkt_response) } -async fn send_tcp_rst( - headers: LaxPacketHeaders<'_>, - socket: &Arc, - remote_socket: SocketAddr, -) { +fn send_tcp_rst(headers: LaxPacketHeaders<'_>) -> Option> { let Some(LinkHeader::Ethernet2(mut ethernet_header)) = headers.link else { - return; + return None; }; std::mem::swap( &mut ethernet_header.source, @@ -142,7 +114,7 @@ async fn send_tcp_rst( .collect(); let Some(NetHeaders::Ipv4(mut ip_header, _)) = headers.net else { - return; + return None; }; ip_header.identification = 0; ip_header.fragment_offset = IpFragOffset::ZERO; @@ -152,7 +124,7 @@ async fn send_tcp_rst( let ip_header_bytes = ip_header.to_bytes(); let Some(TransportHeader::Tcp(mut tcp_header)) = headers.transport else { - return; + return None; }; let src_port_orig = tcp_header.source_port; let seq_num_orig = tcp_header.sequence_number; @@ -187,8 +159,5 @@ async fn send_tcp_rst( &tcp_header_bytes[..], ].concat(); - socket - .send_to(&pkt_response, remote_socket) - .await - .unwrap_or(0); + Some(pkt_response) } diff --git a/members/nullnet-client/src/crypto.rs b/members/nullnet-client/src/crypto.rs new file mode 100644 index 00000000..537bc360 --- /dev/null +++ b/members/nullnet-client/src/crypto.rs @@ -0,0 +1,118 @@ +use aes_gcm::aead::rand_core::RngCore; +use aes_gcm::aead::{Aead, KeyInit, OsRng}; +use aes_gcm::{Aes256Gcm, Key, Nonce}; + +/// AES-256-GCM cipher for one VLAN tunnel's traffic. Wire format produced by +/// [`Self::encrypt`] / consumed by [`Self::decrypt`] is `nonce[12] || ciphertext+tag`. +/// Unlike `nullnet-server`'s cert-at-rest `Encryptor` (keyed once, process-wide, +/// operates on UTF-8 strings), this is constructed per-tunnel from the key the +/// server hands out at `VlanSetup` time and operates on raw Ethernet frames. +pub(crate) struct TunnelCipher { + cipher: Aes256Gcm, +} + +impl TunnelCipher { + pub(crate) fn new(key: &[u8; 32]) -> Self { + Self { + cipher: Aes256Gcm::new(Key::::from_slice(key)), + } + } + + pub(crate) fn encrypt(&self, plaintext: &[u8]) -> Option> { + let mut nonce_bytes = [0u8; 12]; + OsRng.fill_bytes(&mut nonce_bytes); + let mut out = nonce_bytes.to_vec(); + out.extend( + self.cipher + .encrypt(Nonce::from_slice(&nonce_bytes), plaintext) + .ok()?, + ); + Some(out) + } + + pub(crate) fn decrypt(&self, data: &[u8]) -> Option> { + if data.len() < 12 { + return None; + } + let (nonce_bytes, ciphertext) = data.split_at(12); + self.cipher + .decrypt(Nonce::from_slice(nonce_bytes), ciphertext) + .ok() + } +} + +/// Wire framing for the VLAN userspace forwarder (`forward/send.rs`, +/// `forward/receive.rs`): every datagram on the forward socket is +/// `vlan_id[2, big-endian] || nonce[12] || ciphertext+tag`. The vlan_id has +/// to be readable in the clear so the receiver knows which tunnel's key to +/// decrypt with before it can read anything else. +pub(crate) fn seal(vlan_id: u16, cipher: &TunnelCipher, plaintext: &[u8]) -> Option> { + let mut out = vlan_id.to_be_bytes().to_vec(); + out.extend(cipher.encrypt(plaintext)?); + Some(out) +} + +/// Splits a raw forward-socket datagram into its cleartext `vlan_id` and the +/// remaining `nonce || ciphertext+tag` slice, ready for `TunnelCipher::decrypt`. +pub(crate) fn open_vlan_id(datagram: &[u8]) -> Option<(u16, &[u8])> { + if datagram.len() < 2 { + return None; + } + let (vlan_id_bytes, rest) = datagram.split_at(2); + Some(( + u16::from_be_bytes([vlan_id_bytes[0], vlan_id_bytes[1]]), + rest, + )) +} + +#[cfg(test)] +mod tests { + use super::{TunnelCipher, open_vlan_id, seal}; + + #[test] + fn round_trip() { + let cipher = TunnelCipher::new(&[7u8; 32]); + let frame = b"pretend this is an ethernet frame"; + let ct = cipher.encrypt(frame).unwrap(); + assert_ne!(ct, frame); + assert_eq!(cipher.decrypt(&ct).unwrap(), frame); + } + + #[test] + fn nonce_randomizes_ciphertext() { + let cipher = TunnelCipher::new(&[7u8; 32]); + assert_ne!( + cipher.encrypt(b"same").unwrap(), + cipher.encrypt(b"same").unwrap() + ); + } + + #[test] + fn wrong_key_fails() { + let ct = TunnelCipher::new(&[1u8; 32]).encrypt(b"secret").unwrap(); + assert!(TunnelCipher::new(&[2u8; 32]).decrypt(&ct).is_none()); + } + + #[test] + fn truncated_data_fails_without_panicking() { + let cipher = TunnelCipher::new(&[3u8; 32]); + assert!(cipher.decrypt(&[0u8; 5]).is_none()); + } + + #[test] + fn seal_then_open_round_trips_vlan_id_and_plaintext() { + let cipher = TunnelCipher::new(&[9u8; 32]); + let frame = b"ethernet frame payload"; + let datagram = seal(4242, &cipher, frame).unwrap(); + + let (vlan_id, rest) = open_vlan_id(&datagram).unwrap(); + assert_eq!(vlan_id, 4242); + assert_eq!(cipher.decrypt(rest).unwrap(), frame); + } + + #[test] + fn open_vlan_id_rejects_short_datagrams() { + assert!(open_vlan_id(&[0u8]).is_none()); + assert!(open_vlan_id(&[]).is_none()); + } +} diff --git a/members/nullnet-client/src/forward/receive.rs b/members/nullnet-client/src/forward/receive.rs index ad90662a..bc53758a 100644 --- a/members/nullnet-client/src/forward/receive.rs +++ b/members/nullnet-client/src/forward/receive.rs @@ -5,8 +5,10 @@ use tokio::net::UdpSocket; use tokio::sync::RwLock; use tun_rs::AsyncDevice; -use crate::craft::reject_payloads::send_termination_message; +use crate::craft::reject_payloads::build_termination_message; +use crate::crypto; use crate::forward::frame::Frame; +use crate::peers::peer::Peers; /// Handles incoming network packets (receives packets from the socket and sends them to the TAP interface), /// ensuring the firewall rules are correctly observed. @@ -14,29 +16,51 @@ pub async fn receive( device: &Arc, socket: &Arc, firewall: &Arc>, + peers: &Arc>, ) { let mut frame = Frame::new(); let mut remote_socket; loop { - // wait until there is an incoming packet on the socket (packets on the socket are raw IP) + // wait until there is an incoming datagram on the socket let Ok((s, r)) = socket.recv_from(&mut frame.frame).await else { continue; }; (frame.size, remote_socket) = (s, r); if frame.size > 0 { - let pkt_data = frame.pkt_data(); + let datagram = &frame.frame[..frame.size]; + // the vlan_id has to be readable before decryption so we know + // which tunnel's key to decrypt with + let Some((vlan_id, sealed)) = crypto::open_vlan_id(datagram) else { + continue; + }; + let Some(cipher) = peers.read().await.get_key(vlan_id) else { + continue; + }; + // decrypt as the packet exits the tunnel; auth failure (wrong + // key, corrupted/spoofed datagram) drops it here + let Some(pkt_data) = cipher.decrypt(sealed) else { + continue; + }; + match firewall .read() .await - .resolve_packet(pkt_data, FirewallDirection::IN) + .resolve_packet(&pkt_data, FirewallDirection::IN) { FirewallAction::ACCEPT => { // write packet to the kernel - device.send(pkt_data).await.unwrap_or(0); + device.send(&pkt_data).await.unwrap_or(0); } FirewallAction::REJECT => { - send_termination_message(pkt_data, socket, remote_socket).await; + if let Some(reply) = build_termination_message(&pkt_data) + && let Some(reply_datagram) = crypto::seal(vlan_id, &cipher, &reply) + { + socket + .send_to(&reply_datagram, remote_socket) + .await + .unwrap_or(0); + } } FirewallAction::DENY => {} } diff --git a/members/nullnet-client/src/forward/send.rs b/members/nullnet-client/src/forward/send.rs index 2854580b..1ebf27ce 100644 --- a/members/nullnet-client/src/forward/send.rs +++ b/members/nullnet-client/src/forward/send.rs @@ -7,6 +7,7 @@ use tokio::net::UdpSocket; use tokio::sync::RwLock; use tun_rs::AsyncDevice; +use crate::crypto; use crate::forward::frame::Frame; use crate::peers::peer::{Peers, VethKey}; @@ -26,7 +27,7 @@ pub async fn send( if frame.size > 0 { // send the packet to the socket let pkt_data = frame.pkt_data(); - let Ok(dst_socket) = get_dst_socket(pkt_data, &peers).await else { + let Ok((dst_socket, vlan_id)) = get_dst_socket(pkt_data, &peers).await else { continue; }; match firewall @@ -35,7 +36,15 @@ pub async fn send( .resolve_packet(pkt_data, FirewallDirection::OUT) { FirewallAction::ACCEPT => { - socket.send_to(pkt_data, dst_socket).await.unwrap_or(0); + // encrypt as the packet enters the tunnel: without a key + // for this vlan_id (tunnel torn down mid-flight, or + // setup never landed) there is nothing safe to send. + let Some(cipher) = peers.read().await.get_key(vlan_id) else { + continue; + }; + if let Some(datagram) = crypto::seal(vlan_id, &cipher, pkt_data) { + socket.send_to(&datagram, dst_socket).await.unwrap_or(0); + } } FirewallAction::DENY | FirewallAction::REJECT => {} } @@ -43,7 +52,10 @@ pub async fn send( } } -async fn get_dst_socket(pkt_data: &[u8], peers: &Arc>) -> Result { +async fn get_dst_socket( + pkt_data: &[u8], + peers: &Arc>, +) -> Result<(SocketAddr, u16), Error> { let headers = LaxPacketHeaders::from_ethernet(pkt_data).handle_err(location!())?; let vlan_id = headers .vlan_ids() @@ -63,10 +75,11 @@ async fn get_dst_socket(pkt_data: &[u8], peers: &Arc>) -> Result, + /// Per-tunnel AES-256-GCM cipher, keyed by `vlan_id`. Populated from the + /// key the server hands out in `VlanSetup` and used by the userspace + /// forwarder (`forward/send.rs`, `forward/receive.rs`) to encrypt/decrypt + /// this tunnel's traffic. `Arc`-wrapped so callers can clone a handle out + /// without holding the `Peers` lock across the actual crypto work. + keys: HashMap>, } impl Peers { @@ -24,8 +32,17 @@ impl Peers { self.ips.insert(veth_key, eth_ip); } + pub fn insert_key(&mut self, vlan_id: u16, key: &[u8; 32]) { + self.keys.insert(vlan_id, Arc::new(TunnelCipher::new(key))); + } + + pub fn get_key(&self, vlan_id: u16) -> Option> { + self.keys.get(&vlan_id).cloned() + } + pub fn remove(&mut self, vlan_id: u16) { self.ips.retain(|key, _| key.vlan_id != vlan_id); + self.keys.remove(&vlan_id); } } diff --git a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh index 366b7340..eb849330 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-setup.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-setup.sh @@ -1,10 +1,10 @@ #!/bin/bash # Read CLI arguments: -if [ "$#" -lt 7 ] || [ "$#" -gt 8 ]; then - echo "Usage: $0 [docker_container]" - echo "Example (standalone): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104" - echo "Example (docker): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104 my_container" +if [ "$#" -lt 9 ] || [ "$#" -gt 10 ]; then + echo "Usage: $0 [docker_container]" + echo "Example (standalone): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104 <64 hex chars> 20100" + echo "Example (docker): $0 100 ns_100_s 10.0.0.1/29 br_100_s 10.0.0.2/29 192.168.1.102 192.168.1.104 <64 hex chars> 20100 my_container" exit 1 fi @@ -15,7 +15,9 @@ BR_NAME=$4 BR_NET=$5 LOCAL_IP=$6 REMOTE_IP=$7 -DOCKER_CONTAINER=$8 +KEY_HEX=$8 +DSTPORT=$9 +DOCKER_CONTAINER=${10} BR_IP=$(echo $BR_NET | cut -d'/' -f1) @@ -56,7 +58,15 @@ if [ -z "$DOCKER_CONTAINER" ]; then fi if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then - # Same host: connect bridges with a veth pair instead of a VXLAN tunnel + # Same host: connect bridges with a veth pair instead of a VXLAN tunnel. + # This traffic never leaves the host, so there's no physical-network + # sniffer to defend against — but it's still worth encrypting for + # defense-in-depth against another, differently-privileged + # container/process on the SAME host that could otherwise read this + # veth's or bridge's plaintext traffic directly. MACsec (802.1AE) wraps + # the veth link itself in AES-256-GCM, keyed with this tunnel's key — + # no IP addressing involved, so it works regardless of what the + # containers on either side are doing. VETH_S="veth-${VXLAN_ID}-s" VETH_C="veth-${VXLAN_ID}-c" # Both ends are created atomically; the losing task's EEXIST is harmless @@ -64,17 +74,97 @@ if [ "$LOCAL_IP" == "$REMOTE_IP" ]; then # Attach our end to our bridge if [[ "$BR_NAME" == *_s ]]; then LOCAL_VETH="$VETH_S" + PEER_VETH="$VETH_C" + MACSEC_IF="macsec-${VXLAN_ID}-s" else LOCAL_VETH="$VETH_C" + PEER_VETH="$VETH_S" + MACSEC_IF="macsec-${VXLAN_ID}-c" fi - sudo ip link set "$LOCAL_VETH" master "$BR_NAME" - sudo ip link set "$LOCAL_VETH" mtu $OVERLAY_MTU up + + # The peer's MAC is available immediately: `ip link add ... peer name + # ...` creates both ends atomically in one kernel call, whether this + # invocation won the race above or lost it to the sibling script. + PEER_MAC=$(cat /sys/class/net/$PEER_VETH/address) + KEY_ID=$(printf '%032x' $VXLAN_ID) + + # MACsec adds up to 32 bytes of overhead (SecTAG + ICV for GCM-AES-256). + # Give the underlying veth the extra room — it's a virtual, host-only + # link with no physical MTU constraint — so the macsec interface on + # top of it can still carry a full OVERLAY_MTU-sized frame. + sudo ip link set "$LOCAL_VETH" mtu $((OVERLAY_MTU + 32)) up + + # Note the argument order: `port` (part of this device's own SCI) has + # to come before `cipher` — iproute2's macsec option parser is + # positional here, not a free-order keyword scanner, and silently + # rejects `port` if it comes after `cipher` ("unknown command + # \"port\"?"). Unlike the veth-pair creation above, none of these four + # commands race against the sibling script invocation (each side only + # ever touches its own uniquely-named macsec interface), so their + # stderr is deliberately left unsuppressed — a real failure here + # should be loud, not silently swallowed. + sudo ip link add link "$LOCAL_VETH" "$MACSEC_IF" type macsec port 1 cipher gcm-aes-256 encrypt on + sudo ip macsec add "$MACSEC_IF" tx sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" + sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" on + sudo ip macsec add "$MACSEC_IF" rx port 1 address "$PEER_MAC" sa 0 pn 1 on key "$KEY_ID" "$KEY_HEX" + + sudo ip link set "$MACSEC_IF" master "$BR_NAME" + sudo ip link set "$MACSEC_IF" mtu $OVERLAY_MTU up else - # Create the VXLAN tunnel using your physical IP and interface: - sudo ip link add vxlan-$NS_NAME type vxlan id $VXLAN_ID local $LOCAL_IP remote $REMOTE_IP dstport 4789 # dev ens18 + # Create the VXLAN tunnel using your physical IP and interface. Each + # tunnel gets its own dstport (instead of the IANA-standard 4789) so + # the XFRM policies below can tell concurrent tunnels between the same + # host pair apart. + sudo ip link add vxlan-$NS_NAME type vxlan id $VXLAN_ID local $LOCAL_IP remote $REMOTE_IP dstport $DSTPORT # dev ens18 # Attach the VXLAN to the bridge: sudo ip link set vxlan-$NS_NAME master $BR_NAME sudo ip link set vxlan-$NS_NAME mtu $OVERLAY_MTU up + + # Encrypt this tunnel's traffic at the kernel level (AES-256-GCM via + # IPsec/ESP, transport mode) between the two hosts' physical IPs, + # scoped to this tunnel's dstport so it doesn't collide with any other + # concurrent VXLAN tunnel between the same host pair. + # + # RFC4106 GCM keys are "AES key || 4-byte salt". The server only hands + # out a 32-byte AES key (shared verbatim by both VLAN's software AEAD + # and this XFRM SA), so the salt is derived here, identically on both + # ends, from that same key — it doesn't need to be secret on its own, + # only reproducible from the shared secret both sides already have. + SALT_HEX=$(printf '%s' "$KEY_HEX" | sha256sum | cut -c1-8) + # `ip xfrm state add`'s ALGO-KEYMAT requires a "0x" prefix — a bare hex + # string is rejected outright with a bare "RTNETLINK answers: Invalid + # argument", confirmed by extensive live testing (see commit history). + AEAD_KEY_HEX="0x${KEY_HEX}${SALT_HEX}" + # SPI values 1-255 are IANA-reserved (RFC 4301) and the kernel's XFRM + # code rejects them outright ("Invalid argument"). vxlan_id starts at + # 101 (see net_id_pool.rs), which falls straight into that reserved + # range — offset it well clear of 255 rather than using the raw ID. + SPI=$(printf '0x%08x' $((VXLAN_ID + 1000))) + + # Outbound: this host -> remote. + # Note the argument order in both commands below — same lesson as + # the macsec argument-order bug, `ip xfrm` is positional, not a + # free-order keyword scanner: + # - `state add`: the ALGO-LIST (`aead ...`) must come before + # `mode`, not after — "ID [ALGO-LIST] [mode MODE] ..." per + # `ip xfrm state help`. Reversed, it fails with a bare + # "RTNETLINK answers: Invalid argument". + # - `policy add`: the selector (src/dst/proto/dport) must stay + # contiguous, with `dir` only appearing after it's complete — + # "SELECTOR dir DIR ..." per `ip xfrm policy help`. Splitting it + # by putting `dir` in the middle confuses the parser into + # thinking `proto` was given twice ("duplicate \"unknown\": + # \"proto\" is the second value"). + sudo ip xfrm state add src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI \ + aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 mode transport + sudo ip xfrm policy add src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out \ + tmpl src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI mode transport + + # Inbound: remote -> this host. + sudo ip xfrm state add src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI \ + aead 'rfc4106(gcm(aes))' $AEAD_KEY_HEX 128 mode transport + sudo ip xfrm policy add src $REMOTE_IP dst $LOCAL_IP proto udp dport $DSTPORT dir in \ + tmpl src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI mode transport fi # Enable IP forwarding: diff --git a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh index 49a938aa..f64ae11f 100755 --- a/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh +++ b/members/nullnet-client/vxlan_scripts/vxlan-teardown.sh @@ -1,19 +1,42 @@ #!/bin/bash # Read CLI arguments: -if [ "$#" -lt 3 ] || [ "$#" -gt 4 ]; then - echo "Usage: $0 [docker_container]" - echo "Example (standalone): $0 100 ns_100_s br_100_s" - echo "Example (docker): $0 100 ns_100_s br_100_s my_container" +if [ "$#" -lt 6 ] || [ "$#" -gt 7 ]; then + echo "Usage: $0 [docker_container]" + echo "Example (standalone): $0 100 ns_100_s br_100_s 192.168.1.102 192.168.1.104 20100" + echo "Example (docker): $0 100 ns_100_s br_100_s 192.168.1.102 192.168.1.104 20100 my_container" exit 1 fi VXLAN_ID=$1 NS_NAME=$2 BR_NAME=$3 -DOCKER_CONTAINER=$4 +LOCAL_IP=$4 +REMOTE_IP=$5 +DSTPORT=$6 +DOCKER_CONTAINER=$7 -# Remove the VXLAN tunnel or same-host veth pair: +# Remove this tunnel's XFRM state + policy pair, if any was installed (the +# same-host branch of vxlan-setup.sh never creates one). +if [ "$LOCAL_IP" != "$REMOTE_IP" ]; then + # Must match the same offset vxlan-setup.sh uses, to delete the actual + # installed SPI rather than the raw (and IANA-reserved) vxlan_id. + SPI=$(printf '0x%08x' $((VXLAN_ID + 1000))) + # Same argument-order requirement as vxlan-setup.sh: selector fields + # (src/dst/proto/dport) must stay contiguous, with `dir` only after. + sudo ip xfrm policy delete src $LOCAL_IP dst $REMOTE_IP proto udp dport $DSTPORT dir out 2>/dev/null + sudo ip xfrm state delete src $LOCAL_IP dst $REMOTE_IP proto esp spi $SPI 2>/dev/null + sudo ip xfrm policy delete src $REMOTE_IP dst $LOCAL_IP proto udp dport $DSTPORT dir in 2>/dev/null + sudo ip xfrm state delete src $REMOTE_IP dst $LOCAL_IP proto esp spi $SPI 2>/dev/null +fi + +# Remove the VXLAN tunnel or same-host veth pair. Deleting a veth end also +# destroys its peer and cascades to remove any macsec interface stacked on +# either end (the same-host branch of vxlan-setup.sh wraps each end in one), +# but delete both macsec names explicitly too rather than depend solely on +# that cascade. +sudo ip link del macsec-${VXLAN_ID}-s 2>/dev/null +sudo ip link del macsec-${VXLAN_ID}-c 2>/dev/null sudo ip link set vxlan-$NS_NAME down && sudo ip link del vxlan-$NS_NAME sudo ip link set veth-${VXLAN_ID}-s down && sudo ip link del veth-${VXLAN_ID}-s diff --git a/members/nullnet-grpc-lib/proto/nullnet_grpc.proto b/members/nullnet-grpc-lib/proto/nullnet_grpc.proto index b7f3e88f..3df1cd8f 100644 --- a/members/nullnet-grpc-lib/proto/nullnet_grpc.proto +++ b/members/nullnet-grpc-lib/proto/nullnet_grpc.proto @@ -90,6 +90,10 @@ message VlanSetup { string local_ip = 5; string remote_ip = 6; optional HostMapping host_mapping = 7; + // Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + // sent identically to both endpoints. Used to encrypt/decrypt traffic in + // the client's userspace VLAN forwarder (see forward/send.rs, forward/receive.rs). + bytes encryption_key = 8; } message VlanTeardown { @@ -111,6 +115,15 @@ message VxlanSetup { // The receiving client installs DNAT(dnat_port -> overlay_ip) so the // initiator's traffic on that local port is steered into the new VXLAN. optional uint32 dnat_port = 11; + // Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + // sent identically to both endpoints. Used as the XFRM/ESP SA key that + // encrypts this tunnel's traffic at the kernel level. + bytes encryption_key = 12; + // Per-tunnel VXLAN UDP destination port (replaces the IANA-standard 4789 + // default). Each tunnel gets a distinct port so an XFRM policy — which + // selects by src/dst IP and port, not by VNI — can tell concurrent + // tunnels between the same host pair apart. + uint32 dstport = 13; } message VxlanTeardown { @@ -118,6 +131,11 @@ message VxlanTeardown { string ns_name = 2; string br_name = 3; optional string docker_container = 4; + // local_ip/remote_ip/dstport: needed to remove this tunnel's XFRM SA + + // policy pair (same values used to install them in VxlanSetup). + string local_ip = 5; + string remote_ip = 6; + uint32 dstport = 7; } message MsgId { diff --git a/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs b/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs index 8a2d87d4..d38a50c0 100644 --- a/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs +++ b/members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs @@ -63,6 +63,11 @@ pub struct VlanSetup { pub remote_ip: ::prost::alloc::string::String, #[prost(message, optional, tag = "7")] pub host_mapping: ::core::option::Option, + /// Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + /// sent identically to both endpoints. Used to encrypt/decrypt traffic in + /// the client's userspace VLAN forwarder (see forward/send.rs, forward/receive.rs). + #[prost(bytes = "vec", tag = "8")] + pub encryption_key: ::prost::alloc::vec::Vec, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct VlanTeardown { @@ -96,6 +101,17 @@ pub struct VxlanSetup { /// initiator's traffic on that local port is steered into the new VXLAN. #[prost(uint32, optional, tag = "11")] pub dnat_port: ::core::option::Option, + /// Per-tunnel AES-256 key (32 raw bytes), generated once by the server and + /// sent identically to both endpoints. Used as the XFRM/ESP SA key that + /// encrypts this tunnel's traffic at the kernel level. + #[prost(bytes = "vec", tag = "12")] + pub encryption_key: ::prost::alloc::vec::Vec, + /// Per-tunnel VXLAN UDP destination port (replaces the IANA-standard 4789 + /// default). Each tunnel gets a distinct port so an XFRM policy — which + /// selects by src/dst IP and port, not by VNI — can tell concurrent + /// tunnels between the same host pair apart. + #[prost(uint32, tag = "13")] + pub dstport: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct VxlanTeardown { @@ -107,6 +123,14 @@ pub struct VxlanTeardown { pub br_name: ::prost::alloc::string::String, #[prost(string, optional, tag = "4")] pub docker_container: ::core::option::Option<::prost::alloc::string::String>, + /// local_ip/remote_ip/dstport: needed to remove this tunnel's XFRM SA + + /// policy pair (same values used to install them in VxlanSetup). + #[prost(string, tag = "5")] + pub local_ip: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub remote_ip: ::prost::alloc::string::String, + #[prost(uint32, tag = "7")] + pub dstport: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MsgId { diff --git a/members/nullnet-server/src/net.rs b/members/nullnet-server/src/net.rs index 726baba4..fe6d63fb 100644 --- a/members/nullnet-server/src/net.rs +++ b/members/nullnet-server/src/net.rs @@ -17,9 +17,20 @@ pub(crate) trait NetExt { remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: Option, ) -> Option<(Ipv4Addr, NetMessage)>; - fn teardown(self, net_id: u32, side: &str, docker_container: Option) -> NetMessage; + #[allow(clippy::too_many_arguments)] + fn teardown( + self, + net_id: u32, + side: &str, + docker_container: Option, + local_ip: IpAddr, + remote_ip: IpAddr, + dstport: Option, + ) -> NetMessage; } impl NetExt for Net { @@ -32,9 +43,18 @@ impl NetExt for Net { remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: Option, ) -> Option<(Ipv4Addr, NetMessage)> { match self { - Net::Vlan => vlan_setup(msg_id, dest, remote_server_name, net_id, remote), + Net::Vlan => vlan_setup( + msg_id, + dest, + remote_server_name, + net_id, + remote, + encryption_key, + ), Net::Vxlan => vxlan_setup( msg_id, dest, @@ -43,11 +63,21 @@ impl NetExt for Net { remote, docker_containers, dnat_port, + encryption_key, + dstport.unwrap_or(0), ), } } - fn teardown(self, net_id: u32, side: &str, docker_container: Option) -> NetMessage { + fn teardown( + self, + net_id: u32, + side: &str, + docker_container: Option, + local_ip: IpAddr, + remote_ip: IpAddr, + dstport: Option, + ) -> NetMessage { match self { Net::Vlan => NetMessage { message: Some(net_message::Message::VlanTeardown(VlanTeardown { @@ -60,6 +90,9 @@ impl NetExt for Net { ns_name: format!("ns_{net_id}_{side}"), br_name: format!("br_{net_id}_{side}"), docker_container, + local_ip: local_ip.to_string(), + remote_ip: remote_ip.to_string(), + dstport: u32::from(dstport.unwrap_or(0)), })), }, } @@ -73,6 +106,7 @@ fn vlan_setup( remote_server_name: Option, vlan_id: u32, remote: IpAddr, + encryption_key: [u8; 32], ) -> Option<(Ipv4Addr, NetMessage)> { // Map vlan_id to a /30 block within 10.0.0.0/8. // Each ID gets 4 IPs (2 usable), with 2 IPs used for server/client veth. @@ -106,11 +140,13 @@ fn vlan_setup( local_ip: dest.to_string(), remote_ip: remote.to_string(), host_mapping, + encryption_key: encryption_key.to_vec(), })), }, )) } +#[allow(clippy::too_many_arguments)] fn vxlan_setup( msg_id: String, dest: IpAddr, @@ -119,6 +155,8 @@ fn vxlan_setup( remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: u32, ) -> Option<(Ipv4Addr, NetMessage)> { // Map vxlan_id to a /29 block within 10.0.0.0/8. // Each ID gets 8 IPs (6 usable), with 4 IPs used for ns/br server/client. @@ -176,6 +214,8 @@ fn vxlan_setup( host_mapping, docker_container, dnat_port, + encryption_key: encryption_key.to_vec(), + dstport, })), }, )) diff --git a/members/nullnet-server/src/net_id_pool.rs b/members/nullnet-server/src/net_id_pool.rs index 291fa22c..e17f5076 100644 --- a/members/nullnet-server/src/net_id_pool.rs +++ b/members/nullnet-server/src/net_id_pool.rs @@ -1,3 +1,5 @@ +use aes_gcm::aead::OsRng; +use aes_gcm::aead::rand_core::RngCore; use std::collections::BTreeSet; use std::sync::LazyLock; @@ -68,6 +70,61 @@ impl NetIdPool { } } +/// Minimum/maximum allocatable UDP port for per-tunnel VXLAN dstports. +/// Kept out of the IANA ephemeral range (32768-60999) and away from 4789 +/// (the VXLAN default) to avoid colliding with unrelated local sockets. +const MIN_VXLAN_PORT: u16 = 20000; +const MAX_VXLAN_PORT: u16 = 60000; + +/// Pool of per-tunnel UDP destination ports, used so concurrent VXLAN +/// tunnels between the same physical host pair each get a distinct dstport. +/// This is what lets an XFRM policy (which selects by IP + port, not VNI) +/// tell those tunnels apart. Same allocate/free-with-reuse shape as `NetIdPool`. +#[derive(Debug)] +pub(crate) struct UdpPortPool { + next_fresh: u16, + freed: BTreeSet, +} + +impl UdpPortPool { + pub(crate) fn new() -> Self { + Self { + next_fresh: MIN_VXLAN_PORT, + freed: BTreeSet::new(), + } + } + + pub(crate) fn allocate(&mut self) -> Option { + if let Some(&port) = self.freed.iter().next() { + self.freed.remove(&port); + return Some(port); + } + + if self.next_fresh <= MAX_VXLAN_PORT { + let port = self.next_fresh; + self.next_fresh += 1; + Some(port) + } else { + None + } + } + + pub(crate) fn free(&mut self, port: u16) { + if (MIN_VXLAN_PORT..=MAX_VXLAN_PORT).contains(&port) { + self.freed.insert(port); + } + } +} + +/// Generate a fresh random 32-byte AES-256 key for one tunnel. Called once +/// per net_id allocation; the same bytes are sent to both endpoints so they +/// share a single symmetric key for that tunnel only. +pub(crate) fn generate_key() -> [u8; 32] { + let mut key = [0u8; 32]; + OsRng.fill_bytes(&mut key); + key +} + #[cfg(test)] impl NetIdPool { /// Number of IDs currently in use (allocated but not freed). @@ -187,4 +244,56 @@ mod tests { let free = total - in_use; assert_eq!(total, in_use + free); } + + #[test] + fn test_udp_port_pool_allocate_sequential() { + let mut pool = UdpPortPool::new(); + assert_eq!(pool.allocate(), Some(MIN_VXLAN_PORT)); + assert_eq!(pool.allocate(), Some(MIN_VXLAN_PORT + 1)); + assert_eq!(pool.allocate(), Some(MIN_VXLAN_PORT + 2)); + } + + #[test] + fn test_udp_port_pool_reuse_freed() { + let mut pool = UdpPortPool::new(); + let p1 = pool.allocate().unwrap(); + let p2 = pool.allocate().unwrap(); + pool.allocate(); + + pool.free(p2); + pool.free(p1); + + assert_eq!(pool.allocate(), Some(p1)); + assert_eq!(pool.allocate(), Some(p2)); + } + + #[test] + fn test_udp_port_pool_exhaustion() { + let mut pool = UdpPortPool::new(); + pool.next_fresh = MAX_VXLAN_PORT; + + assert_eq!(pool.allocate(), Some(MAX_VXLAN_PORT)); + assert_eq!(pool.allocate(), None); + + pool.free(MAX_VXLAN_PORT); + assert_eq!(pool.allocate(), Some(MAX_VXLAN_PORT)); + assert_eq!(pool.allocate(), None); + } + + #[test] + fn test_udp_port_pool_free_ignores_out_of_range() { + let mut pool = UdpPortPool::new(); + pool.free(0); + pool.free(MIN_VXLAN_PORT - 1); + pool.free(MAX_VXLAN_PORT + 1); + assert!(pool.freed.is_empty()); + } + + #[test] + fn test_generate_key_is_random_and_full_length() { + let k1 = generate_key(); + let k2 = generate_key(); + assert_eq!(k1.len(), 32); + assert_ne!(k1, k2); + } } diff --git a/members/nullnet-server/src/nullnet_grpc_impl.rs b/members/nullnet-server/src/nullnet_grpc_impl.rs index 935f4fdf..0215862f 100644 --- a/members/nullnet-server/src/nullnet_grpc_impl.rs +++ b/members/nullnet-server/src/nullnet_grpc_impl.rs @@ -1,6 +1,7 @@ use crate::env::NET_TYPE; use crate::events::Event; use crate::graphviz::generate_graphviz; +use crate::net_id_pool::generate_key; use crate::orchestrator::Orchestrator; use crate::services::changes::{ apply_changes, collect_dep_chain_edges, detect_services_list_changes, @@ -12,9 +13,9 @@ use crate::services::service_info::{ServiceInfo, backend_involved_services}; use crate::timeout::check_timeouts; use nullnet_grpc_lib::nullnet_grpc::nullnet_grpc_server::NullnetGrpc; use nullnet_grpc_lib::nullnet_grpc::{ - AgentEvent, BackendTriggerRequest, CertBundle, Empty, MsgId, NetMessage, NetType, PortMapping, - PortMappingBundle, ProxyRequest, ServiceTrigger, Services, ServicesListResponse, Upstream, - agent_event::Event as AgentEventKind, + AgentEvent, BackendTriggerRequest, CertBundle, Empty, MsgId, Net, NetMessage, NetType, + PortMapping, PortMappingBundle, ProxyRequest, ServiceTrigger, Services, ServicesListResponse, + Upstream, agent_event::Event as AgentEventKind, }; use nullnet_liberror::{Error, ErrorHandler, Location, location}; use std::collections::{HashMap, HashSet}; @@ -868,6 +869,31 @@ impl NullnetGrpcImpl { .await; } + // One AES-256 key per tunnel, handed identically to both + // endpoints below. For VXLAN, also reserve a per-tunnel UDP + // dstport so the two hosts' XFRM policies can tell this + // tunnel apart from any other concurrent tunnel between the + // same physical host pair. + let encryption_key = generate_key(); + let dstport = if *NET_TYPE == Net::Vxlan { + match orchestrator.allocate_vxlan_port(net_id).await { + Some(port) => Some(u32::from(port)), + None => { + eprintln!("UDP port pool exhausted"); + orchestrator.free_net_id(net_id).await; + if let Some(stack_map) = services.write().await.get_mut(&stack) + && let Some(ServiceInfo::Registered(reg)) = + stack_map.get_mut(server.name()) + { + reg.remove_client(&client); + } + return EdgeOutcome::Failed; + } + } + } else { + None + }; + let orch = orchestrator.clone(); let cd = client_docker.clone(); let sd = server_docker.clone(); @@ -878,6 +904,8 @@ impl NullnetGrpcImpl { client_ethernet, (cd, sd), None, + encryption_key, + dstport, ); let orch2 = orchestrator.clone(); let cd = client_docker.clone(); @@ -889,6 +917,8 @@ impl NullnetGrpcImpl { server_ethernet, (cd, sd), backend_entry_port, + encryption_key, + dstport, ); let (server_ok, client_ok) = tokio::join!(server_res, client_res); diff --git a/members/nullnet-server/src/orchestrator.rs b/members/nullnet-server/src/orchestrator.rs index 446c09f0..ad4d7e4c 100644 --- a/members/nullnet-server/src/orchestrator.rs +++ b/members/nullnet-server/src/orchestrator.rs @@ -1,7 +1,7 @@ use crate::env::NET_TYPE; use crate::events::{Event, EventStore}; use crate::net::NetExt; -use crate::net_id_pool::NetIdPool; +use crate::net_id_pool::{NetIdPool, UdpPortPool}; use crate::services::changes::{apply_changes, detect_node_disconnect_changes}; use crate::services::input::StackMap; use nullnet_grpc_lib::nullnet_grpc::{ @@ -23,6 +23,12 @@ pub struct Orchestrator { clients: Arc>>, pending: Arc>>>, net_id_pool: Arc>, + /// Per-tunnel VXLAN UDP dstport pool. Unused in VLAN mode. + udp_port_pool: Arc>, + /// net_id -> allocated dstport, for VXLAN tunnels only. Lets + /// `send_net_teardown` free the port without every call site having to + /// thread it through. + net_id_ports: Arc>>, pub(crate) events: EventStore, } @@ -32,6 +38,8 @@ impl Orchestrator { clients: Arc::new(RwLock::new(HashMap::new())), pending: Arc::new(Mutex::new(HashMap::new())), net_id_pool: Arc::new(Mutex::new(NetIdPool::new())), + udp_port_pool: Arc::new(Mutex::new(UdpPortPool::new())), + net_id_ports: Arc::new(Mutex::new(HashMap::new())), events: EventStore::new(), } } @@ -99,6 +107,7 @@ impl Orchestrator { } } + #[allow(clippy::too_many_arguments)] pub(crate) async fn send_net_setup( &self, dest: IpAddr, @@ -107,6 +116,8 @@ impl Orchestrator { remote: IpAddr, docker_containers: (Option, Option), dnat_port: Option, + encryption_key: [u8; 32], + dstport: Option, ) -> Option { let outbound = self.clients.read().await.get(&dest).cloned(); if let Some(outbound) = outbound { @@ -122,6 +133,8 @@ impl Orchestrator { remote, docker_containers, dnat_port, + encryption_key, + dstport, )?; if outbound.send(Ok(message)).await.is_err() { @@ -197,6 +210,22 @@ impl Orchestrator { self.net_id_pool.lock().await.allocate() } + /// Release a `net_id` that was allocated but never dispatched to either + /// endpoint (e.g. a follow-up allocation failed). No teardown messages + /// are sent — nothing was ever set up on either client. + pub(crate) async fn free_net_id(&self, net_id: u32) { + self.net_id_pool.lock().await.free(net_id); + } + + /// Allocate a per-tunnel VXLAN dstport and remember it against `net_id` + /// so `send_net_teardown` can free it later without the caller having to + /// carry it around. Only meaningful when `NET_TYPE == Net::Vxlan`. + pub(crate) async fn allocate_vxlan_port(&self, net_id: u32) -> Option { + let port = self.udp_port_pool.lock().await.allocate()?; + self.net_id_ports.lock().await.insert(net_id, port); + Some(port) + } + pub(crate) async fn connected_node_ips(&self) -> Vec { self.clients.read().await.keys().copied().collect() } @@ -213,17 +242,27 @@ impl Orchestrator { server_docker: Option, net_id: u32, ) { - for (dest, side, docker) in [(client, "c", client_docker), (server, "s", server_docker)] { + // Peeked (not removed yet) so both teardown messages can carry the + // same dstport that was used to install this tunnel's XFRM state; + // the pool slot itself is freed below, after both sides are notified. + let dstport = self.net_id_ports.lock().await.get(&net_id).copied(); + for (dest, remote, side, docker) in [ + (client, server, "c", client_docker), + (server, client, "s", server_docker), + ] { let outbound = self.clients.read().await.get(&dest).cloned(); if let Some(outbound) = outbound { println!("Sending network {net_id} teardown to client {dest}"); - let message = NET_TYPE.teardown(net_id, side, docker); + let message = NET_TYPE.teardown(net_id, side, docker, dest, remote, dstport); let _ = outbound.send(Ok(message)).await.handle_err(location!()); } } self.net_id_pool.lock().await.free(net_id); + if let Some(port) = self.net_id_ports.lock().await.remove(&net_id) { + self.udp_port_pool.lock().await.free(port); + } } }