windows: Fix NLMTU for ipv6 (#1871)

.github/workflows/smoke/smoke-windows.ps1

@@ -51,15 +51,19 @@ wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null
 $DevName = 'nebula-smoke'
 $Ip1 = '192.168.241.1'
 $Ip2 = '192.168.241.2'
+# Dual stack on purpose: a v4-only overlay never exercises the v6 side of tun.mtu.
+$Ip6_1 = 'fd42:4242:241::1'
+$Ip6_2 = 'fd42:4242:241::2'
+$Mtu = 1300
 $Port = 4242
 
 & $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key"
 if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" }
 
-& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
+& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24,$Ip6_1/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
 if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" }
 
-& $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
+& $NebulaCert sign -name 'peer' -networks "$Ip2/24,$Ip6_2/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
 if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" }
 
 # Windows lighthouse config.
@@ -82,7 +86,7 @@ tun:
   drop_local_broadcast: false
   drop_multicast: false
   tx_queue: 500
-  mtu: 1300
+  mtu: $Mtu
   network_category: private
 logging:
   level: info
@@ -126,7 +130,7 @@ tun:
   drop_local_broadcast: false
   drop_multicast: false
   tx_queue: 500
-  mtu: 1300
+  mtu: $Mtu
 logging:
   level: info
   format: text
@@ -169,7 +173,7 @@ Write-Host '=== WSL diagnostic ==='
 wsl --version 2>&1 | Out-Host
 wsl --list --verbose 2>&1 | Out-Host
 wsl -d $Distro -u root -- uname -a | Out-Host
-wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun"
+wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; { echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6; echo 0 > /proc/sys/net/ipv6/conf/default/disable_ipv6; } 2>/dev/null || true; ls -l /dev/net/tun"
 if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" }
 
 # Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf
@@ -214,6 +218,16 @@ try {
     }
     Write-Host "OK: $DevName NetworkCategory=Private"
 
+    # v6 silently kept the adapter default of 65535 while v4 was correct.
+    foreach ($family in @('IPv4', 'IPv6')) {
+        Wait-Until -TimeoutSec 30 -What "$DevName $family NlMtu=$Mtu" -Predicate {
+            if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before $family mtu was set" }
+            $rows = @(Get-NetIPInterface -InterfaceAlias $DevName -AddressFamily $family -ErrorAction SilentlyContinue)
+            $rows.Count -gt 0 -and -not ($rows | Where-Object { $_.NlMtu -ne $Mtu })
+        }
+        Write-Host "OK: $DevName $family NlMtu=$Mtu"
+    }
+
     Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate {
         if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" }
         $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes"
@@ -221,6 +235,13 @@ try {
     }
     Write-Host "OK: WSL nebula1 has $Ip2"
 
+    Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip6_2" -Predicate {
+        if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 address was up" }
+        $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet6 $Ip6_2' && echo yes"
+        ("$r").Trim() -eq 'yes'
+    }
+    Write-Host "OK: WSL nebula1 has $Ip6_2"
+
     Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate {
         if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" }
         $r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK"
@@ -234,6 +255,14 @@ try {
     }
     Write-Host "OK: windows lighthouse -> WSL peer"
 
+    # Otherwise the v6 networks only prove the interface exists, not that it forwards.
+    Wait-Until -TimeoutSec 30 -What "v6 ping from WSL peer to windows lighthouse ($Ip6_1)" -Predicate {
+        if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 ping succeeded" }
+        $r = wsl -d $Distro -u root -- bash -c "ping -6 -c1 -W1 $Ip6_1 >/dev/null 2>&1 && echo OK"
+        ("$r").Trim() -eq 'OK'
+    }
+    Write-Host "OK: WSL peer -> windows lighthouse over v6"
+
     Write-Host ''
     Write-Host 'All smoke checks passed.'
 }

overlay/tun_windows.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
@@ -182,6 +183,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
 	luid := winipcfg.LUID(t.tun.LUID())
 	routes := *t.Routes.Load()
 	foundDefault4 := false
+	carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
 
 	for _, r := range routes {
 		if len(r.Via) == 0 || !r.Install {
@@ -189,6 +191,9 @@ func (t *winTun) addRoutes(logErrors bool) error {
 			continue
 		}
 
+		// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
+		carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
+
 		// Add our unsafe route as an on-link route to the nebula tun device.
 		err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
 		if err != nil {
@@ -210,6 +215,11 @@ func (t *winTun) addRoutes(logErrors bool) error {
 		}
 	}
 
+	return t.setMTU(luid, foundDefault4, carriesV6)
+}
+
+// setMTU applies tun.mtu per address family. The default route metric rides along on the v4 handle.
+func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error {
 	ipif, err := luid.IPInterface(windows.AF_INET)
 	if err != nil {
 		return fmt.Errorf("failed to get ip interface: %w", err)
@@ -224,6 +234,25 @@ func (t *winTun) addRoutes(logErrors bool) error {
 	if err := ipif.Set(); err != nil {
 		return fmt.Errorf("failed to set ip interface: %w", err)
 	}
+
+	// Windows tracks NLMTU per family and wintun sets neither, so v6 keeps the adapter default of 65535.
+	// Gated so a v4-only overlay under 1280 boots; a v6 one deliberately does not, as linux also refuses.
+	if !carriesV6 {
+		return nil
+	}
+
+	ipif6, err := luid.IPInterface(windows.AF_INET6)
+	if err != nil {
+		// No v6 on the adapter means there is no NLMTU to get wrong. A failed Set below is not the same thing.
+		t.l.Info("Skipping ipv6 MTU, no ipv6 interface on this adapter", "error", err)
+		return nil
+	}
+
+	ipif6.NLMTU = uint32(t.MTU)
+	if err := ipif6.Set(); err != nil {
+		return fmt.Errorf("failed to set ipv6 interface: %w", err)
+	}
+
 	return nil
 }
 

udp/udp_rio_windows.go

@@ -31,7 +31,8 @@ func procyield(cycles uint32)
 
 const (
 	packetsPerRing = 1024
-	bytesPerPacket = 2048 - 32
+	// Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket.
+	bytesPerPacket = MTU
 	receiveSpins   = 15
 )
 
@@ -69,12 +70,14 @@ func NewRIOListener(l *slog.Logger, addr netip.Addr, port int) (*RIOConn, error)
 
 	err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port})
 	if err != nil {
+		u.close()
 		return nil, fmt.Errorf("bind: %w", err)
 	}
 
 	for i := 0; i < packetsPerRing; i++ {
 		err = u.insertReceiveRequest()
 		if err != nil {
+			u.close()
 			return nil, fmt.Errorf("init rx ring: %w", err)
 		}
 	}
@@ -356,15 +359,25 @@ func (u *RIOConn) Close() error {
 		return nil
 	}
 
+	u.close()
+	return nil
+}
+
+// Also unwinds a partial build from NewRIOListener, where isOpen is false and Close would no-op.
+// Socket first, unlike wireguard-go: receive() re-arms every slot, so freeing the rings under a live socket
+// hands the kernel freed pages for all packetsPerRing outstanding receives.
+func (u *RIOConn) close() {
+	// WSASocket reports failure as InvalidHandle, not zero.
+	if u.sock != 0 && u.sock != windows.InvalidHandle {
+		windows.CloseHandle(u.sock)
+	}
+	u.sock = 0
+
 	windows.PostQueuedCompletionStatus(u.rx.iocp, 0, 0, nil)
 	windows.PostQueuedCompletionStatus(u.tx.iocp, 0, 0, nil)
 
 	u.rx.CloseAndZero()
 	u.tx.CloseAndZero()
-	if u.sock != 0 {
-		windows.CloseHandle(u.sock)
-	}
-	return nil
 }
 
 func (ring *ringBuffer) Push() *ringPacket {