Encrypt VLAN/VXLAN tunnel traffic with per-tunnel AES-256 keys

README.md

@@ -39,6 +39,7 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip
   NET_TYPE=VXLAN
   CERT_ENCRYPTION_KEY=<32 raw bytes or 64 hex chars>
   PROXY_IP=192.168.1.100
+  ENCRYPTION_ENABLED=true
   ```
   `CERT_ENCRYPTION_KEY` is **required** — the server refuses to start without it. It encrypts
   TLS certificate private keys (and the DNS-provider credentials of ACME-issued certs) at rest;
@@ -50,6 +51,10 @@ The repository should be cloned under `/root` so the provided `setup-*.sh` scrip
   a per-initiator egress edge to this host. If unset, egress is disabled (the trigger is rejected
   with "PROXY_IP is not configured") — ingress still works.
 
+  `ENCRYPTION_ENABLED` toggles per-tunnel VLAN/VXLAN encryption (AES-256-GCM for VLAN, XFRM/MACsec
+  for VXLAN) and **defaults to `true`** — omit it to keep encryption on. Set it to `false`/`0`/`no`
+  to run tunnels unencrypted instead (a bare vxlan/veth link, no XFRM SA/policy or MACsec).
+
 - TLS certificates are issued from Let's Encrypt via a DNS-01 challenge (UI: *Certificates* page).
   Each cert stores its DNS-provider credentials encrypted at rest and is **renewed automatically**
   before expiry. The renewal scan is tunable via optional env vars (defaults shown):

ebpf/src/main.rs

@@ -25,6 +25,16 @@ use network_types::{
 //   - ARP                          (next-hop resolution)
 //   - TCP to/from SERVER_IP:PORT   (nullnet control plane / gRPC)
 //   - UDP 4789/9999 to/from a peer (nullnet data plane: VXLAN / forward)
+//   - UDP on a live tunnel's per-tunnel VXLAN dstport, to/from the SPECIFIC
+//     peer that port was allocated to (each VXLAN tunnel gets its own
+//     dynamically-allocated dstport instead of the shared 4789 default;
+//     VXLAN_PORTS maps port -> peer IP, populated/depopulated by the control
+//     channel alongside PEERS as tunnels come and go). Paired rather than two
+//     independent sets, so one tunnel's port can't be satisfied by a
+//     different concurrent tunnel's peer IP.
+//   - ESP (proto 50) to/from a peer: cross-host VXLAN tunnels are wrapped in
+//     kernel IPsec/ESP (see vxlan-setup.sh); ESP is portless like ICMP, so it's
+//     scoped to known peers instead of a port check
 // Stateful additions:
 //   - any packet whose flow is already in the CT map is allowed (established
 //     return); every allowed non-ARP packet (re)inserts its canonical 5-tuple,
@@ -45,11 +55,22 @@ const FORWARD_PORT: u16 = 9999;
 
 const PROTO_TCP: u8 = 6;
 const PROTO_UDP: u8 = 17;
+const PROTO_ESP: u8 = 50;
 
 // Allowlist of peer underlay IPs (host-order `u32::from(Ipv4Addr)` keys).
 #[map]
 static PEERS: HashMap<u32, u8> = HashMap::with_max_entries(4096, 0);
 
+// Per-tunnel VXLAN dstport -> the specific peer underlay IP it was allocated
+// to (host-order `u32::from(Ipv4Addr)`, same encoding as PEERS). Each live
+// tunnel gets its own dynamically-allocated dstport (see nullnet-server's
+// `UdpPortPool`) instead of the shared 4789 default, so this is what lets
+// `data_plane()` recognize that traffic without widening the static
+// ALLOW_PORTS operator policy — keyed by port -> peer (not two independent
+// sets) so a packet must match the port its OWN tunnel was assigned.
+#[map]
+static VXLAN_PORTS: HashMap<u16, u32> = HashMap::with_max_entries(4096, 0);
+
 // Per-direction, per-proto destination-port allowlist. Key packs direction and
 // protocol alongside the port (see `allow_key`) so ingress/egress and TCP/UDP of
 // the same number are distinct. Populated by userspace from the four
@@ -145,6 +166,10 @@ fn try_firewall(ctx: &TcContext, is_egress: bool) -> Result<i32, ()> {
                 // ICMP is portless and always allowed (echo + PMTUD/errors, both
                 // directions). Not CT-tracked. Simpler than gating it for now.
                 IpProto::Icmp => return Ok(TC_ACT_OK),
+                // ESP (cross-host VXLAN IPsec) is portless too, but unlike ICMP
+                // it carries real tunnel data, so it stays peer-scoped rather
+                // than unconditionally allowed — see the PROTO_ESP check below.
+                IpProto::Esp => (PROTO_ESP, 0u16, 0u16),
                 _ => return Ok(TC_ACT_SHOT),
             };
 
@@ -161,7 +186,7 @@ fn try_firewall(ctx: &TcContext, is_egress: bool) -> Result<i32, ()> {
 }
 
 // Base (stateless) allow decision, scoped by direction and node role.
-#[inline]
+#[inline(always)]
 fn base_allow(
     is_egress: bool,
     proto: u8,
@@ -176,6 +201,9 @@ fn base_allow(
     if proto == PROTO_UDP && data_plane(src, dst, src_port, dst_port) {
         return true;
     }
+    if proto == PROTO_ESP && (is_peer(src) || is_peer(dst)) {
+        return true;
+    }
     // Gateway outbound: it is the internet boundary — allow all, track it.
     if is_gateway() && is_egress {
         return true;
@@ -193,21 +221,36 @@ fn control_plane(src: u32, dst: u32, src_port: u16, dst_port: u16) -> bool {
     (dst == server && dst_port == ctrl_port) || (src == server && src_port == ctrl_port)
 }
 
-// Data plane: UDP on the VXLAN (4789) or forward (9999) port with a known peer.
+// Data plane: UDP on the VXLAN (4789) or forward (9999) shared ports with any
+// known peer, OR on a live tunnel's own per-tunnel dstport with the SPECIFIC
+// peer that port was allocated to. The shared ports aren't tunnel-exclusive
+// (FORWARD_PORT is one listening socket for every VLAN tunnel on this host),
+// so "any known peer" is correct there; a per-tunnel dstport is exclusive to
+// one tunnel, so it's paired with that tunnel's own peer instead of checked
+// against the flat peer set — otherwise a packet spoofing a *different*
+// concurrent tunnel's peer IP would satisfy this tunnel's port.
 #[inline]
 fn data_plane(src: u32, dst: u32, src_port: u16, dst_port: u16) -> bool {
-    let on_data_port = dst_port == VXLAN_PORT
+    let on_fixed_port = dst_port == VXLAN_PORT
         || src_port == VXLAN_PORT
         || dst_port == FORWARD_PORT
         || src_port == FORWARD_PORT;
-    on_data_port && (is_peer(src) || is_peer(dst))
+    if on_fixed_port && (is_peer(src) || is_peer(dst)) {
+        return true;
+    }
+    vxlan_port_peer(dst_port) == Some(src) || vxlan_port_peer(src_port) == Some(dst)
 }
 
 #[inline]
 fn is_peer(ip: u32) -> bool {
     unsafe { PEERS.get(&ip) }.is_some()
 }
 
+#[inline]
+fn vxlan_port_peer(port: u16) -> Option<u32> {
+    unsafe { VXLAN_PORTS.get(&port) }.copied()
+}
+
 // Is `port` allowed as a destination in this direction/proto? (ALLOW_PORTS key
 // layout must match userspace `allow_key` in members/nullnet-client/src/ebpf.)
 #[inline]

members/nullnet-client/Cargo.toml

@@ -22,5 +22,6 @@ network-interface = "2.0.5"
 gag.workspace = true
 chrono.workspace = true
 nfq = "0.2"
+aes-gcm = "0.10"
 aya = "0.13"
-libc = "0.2"
+libc = "0.2"

members/nullnet-client/src/commands/mod.rs

@@ -26,11 +26,20 @@ pub(crate) async fn setup_br0(rtnetlink_handle: &RtNetLinkHandle) {
     // delete existing OpenFlow rules
     OvsCommand::DeleteFlows.execute();
 
-    // use the built-in switching logic
-    OvsCommand::AddFlow.execute();
-
-    // add our TAP to the bridge as a trunk port
+    // add our TAP to the bridge as a trunk port first, so the flow rules
+    // below can reference it by name
     OvsCommand::AddTrunkPort.execute();
+
+    // Safe fallback (same as OVS's original single default rule) for
+    // anything not covered by a more specific rule — mainly the brief
+    // window between an access port being created and its own redirect
+    // rule (installed in `configure_access_port`) landing.
+    OvsCommand::AddDefaultFlow.execute();
+
+    // Traffic arriving from the trunk (i.e. already decrypted by
+    // nullnet-client's userspace forwarder) is delivered by normal
+    // VLAN-aware switching.
+    OvsCommand::AddTrunkDeliveryFlow.execute();
 }
 
 pub(crate) async fn configure_access_port(
@@ -52,9 +61,22 @@ pub(crate) async fn configure_access_port(
 
     // add the peer interface to the bridge as an access port
     OvsCommand::AddAccessPort(&veth_peer_name, vlan_id).execute();
+
+    // Redirect this port's traffic to the trunk instead of letting OVS
+    // switch it directly to another local access port — and re-add the
+    // 802.1Q tag that gets stripped along the way, since the raw `output`
+    // action used to reach the trunk doesn't do that automatically the way
+    // `actions=normal` would.
+    OvsCommand::AddAccessRedirectFlow(&veth_peer_name, vlan_id).execute();
 }
 
 pub(crate) async fn remove_vlan(rtnetlink_handle: &RtNetLinkHandle, vlan_id: u16) {
+    // remove this port's redirect flow before the port itself disappears,
+    // so no stale rule is left behind that could later match a different,
+    // unrelated port reusing the same OVS port number
+    let veth_peer_name = format!("veth-{vlan_id}p");
+    OvsCommand::DeleteAccessRedirectFlow(&veth_peer_name).execute();
+
     // delete the veth pair
     rtnetlink_handle
         .execute(NetLinkCommand::DeleteVeth(vlan_id))

members/nullnet-client/src/commands/netlink.rs

@@ -8,6 +8,15 @@ use rtnetlink::packet_route::route::{RouteAttribute, RouteHeader};
 use rtnetlink::{Handle, LinkUnspec, LinkVeth, RouteMessageBuilder};
 use std::net::{IpAddr, Ipv4Addr};
 
+/// Matches `vxlan-setup.sh`'s `OVERLAY_MTU`: this environment's physical
+/// underlay (likely the host's own SDN/overlay layer) adds enough hidden
+/// overhead that a full 1500-byte frame gets silently dropped past a
+/// measured ~1104-byte ceiling. VLAN tunnels share the same physical path,
+/// so the veth pair carrying a tunnel's traffic needs the same shrink —
+/// otherwise a large enough frame (e.g. an SSH KEX packet) vanishes with no
+/// error on either end.
+const VLAN_VETH_MTU: u32 = 1080;
+
 #[derive(Debug)]
 pub(super) enum NetLinkCommand<'a> {
     HandleVethPairCreation(Ipv4Network, &'a str, &'a str),
@@ -68,9 +77,9 @@ async fn handle_veth_pair_creation(
     let veth = get_link_by_name(handle, veth_name).await?;
     let veth_peer = get_link_by_name(handle, veth_peer_name).await?;
 
-    // set both ends of the veth pair up
+    // shrink both ends to VLAN_VETH_MTU and set them up
     for link in [&veth, &veth_peer] {
-        set_link_up(handle, link).await?;
+        set_link_mtu_up(handle, link, VLAN_VETH_MTU).await?;
     }
 
     // assign the IP address to veth_name
@@ -214,6 +223,21 @@ async fn set_link_up(handle: &Handle, link: &LinkMessage) -> Result<(), Error> {
     Ok(())
 }
 
+async fn set_link_mtu_up(handle: &Handle, link: &LinkMessage, mtu: u32) -> Result<(), Error> {
+    let req = LinkUnspec::new_with_index(link.header.index)
+        .mtu(mtu)
+        .up()
+        .build();
+    handle
+        .link()
+        .set(req)
+        .execute()
+        .await
+        .handle_err(location!())?;
+
+    Ok(())
+}
+
 async fn delete_link(handle: &Handle, link: LinkMessage) -> Result<(), Error> {
     handle
         .link()

members/nullnet-client/src/commands/ovs.rs

@@ -7,7 +7,30 @@ pub(super) enum OvsCommand<'a> {
     DeleteBridge,
     AddBridge,
     DeleteFlows,
-    AddFlow,
+    /// Fallback for anything not covered by a more specific rule below —
+    /// same behavior as OVS's original single default flow. Mainly covers
+    /// the brief startup window between an access port being created and
+    /// its own redirect flow (below) landing.
+    AddDefaultFlow,
+    /// Traffic arriving from the trunk (already decrypted by nullnet-client's
+    /// userspace forwarder) gets delivered by normal VLAN-aware L2 switching.
+    AddTrunkDeliveryFlow,
+    /// One rule per access port, installed alongside it: redirect this
+    /// port's traffic to the trunk instead of letting OVS switch it
+    /// directly to another local access port (which would bypass the TAP
+    /// and the encrypting userspace forwarder entirely when a tunnel's two
+    /// endpoints happen to be colocated on this host). `output:<port>` is a
+    /// raw action — unlike `actions=normal`, it does *not* re-add the
+    /// 802.1Q tag that access ports carry only internally, so this
+    /// explicitly pushes the tag back on first: without that, packets would
+    /// arrive at nullnet-client's TAP already stripped of their VLAN tag
+    /// and get silently dropped as malformed.
+    AddAccessRedirectFlow(&'a str, u16),
+    /// Removes exactly the rule `AddAccessRedirectFlow` installed for this
+    /// port, so a torn-down tunnel doesn't leave a stale flow entry that
+    /// could wrongly match a future, unrelated port reusing the same
+    /// OVS port number.
+    DeleteAccessRedirectFlow(&'a str),
     AddTrunkPort,
     AddAccessPort(&'a str, u16),
 }
@@ -33,7 +56,11 @@ impl OvsCommand<'_> {
             | OvsCommand::DeleteBridge
             | OvsCommand::AddAccessPort(_, _)
             | OvsCommand::AddTrunkPort => "ovs-vsctl",
-            OvsCommand::DeleteFlows | OvsCommand::AddFlow => "ovs-ofctl",
+            OvsCommand::DeleteFlows
+            | OvsCommand::AddDefaultFlow
+            | OvsCommand::AddTrunkDeliveryFlow
+            | OvsCommand::AddAccessRedirectFlow(_, _)
+            | OvsCommand::DeleteAccessRedirectFlow(_) => "ovs-ofctl",
         }
     }
 
@@ -45,10 +72,36 @@ impl OvsCommand<'_> {
                 .iter()
                 .map(ToString::to_string)
                 .collect(),
-            OvsCommand::AddFlow => ["add-flow", "br0", "priority=0,actions=normal"]
+            OvsCommand::AddDefaultFlow => ["add-flow", "br0", "priority=0,actions=normal"]
                 .iter()
                 .map(ToString::to_string)
                 .collect(),
+            OvsCommand::AddTrunkDeliveryFlow => [
+                "add-flow",
+                "br0",
+                &format!("priority=200,in_port={TAP_NAME},actions=normal"),
+            ]
+            .iter()
+            .map(ToString::to_string)
+            .collect(),
+            OvsCommand::AddAccessRedirectFlow(dev, vlan) => [
+                "-O",
+                "OpenFlow13",
+                "add-flow",
+                "br0",
+                &format!(
+                    "priority=150,in_port={dev},actions=push_vlan:0x8100,mod_vlan_vid:{vlan},output:{TAP_NAME}"
+                ),
+            ]
+            .iter()
+            .map(ToString::to_string)
+            .collect(),
+            OvsCommand::DeleteAccessRedirectFlow(dev) => {
+                ["del-flows", "br0", &format!("in_port={dev}")]
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect()
+            }
             OvsCommand::AddTrunkPort => ["add-port", "br0", TAP_NAME]
                 .iter()
                 .map(ToString::to_string)