//go:build !e2e_testing package overlay import ( "crypto" "errors" "fmt" "log/slog" "net/netip" "os" "path/filepath" "runtime" "slices" "sync/atomic" "syscall" "unsafe" "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" "github.com/slackhq/nebula/wintun" "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" ) type closer interface { Close() } const tunGUIDLabel = "Fixed Nebula Windows GUID v1" type winTun struct { Device string vpnNetworks []netip.Prefix MTU int Routes atomic.Pointer[[]Route] routeTree atomic.Pointer[bart.Table[routing.Gateways]] guid windows.GUID networkCategory networkCategory setCategory bool bypassWDF bool wdfBypass closer l *slog.Logger tun *wintun.NativeTun } func (t *winTun) Read(b []byte) (int, error) { return t.tun.Read(b, 0) } func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) { return nil, fmt.Errorf("newTunFromFd not supported in Windows") } func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) { err := checkWinTunExists() if err != nil { return nil, fmt.Errorf("can not load the wintun driver: %w", err) } deviceName := c.GetString("tun.dev", "") guid, err := generateGUIDByDeviceName(deviceName) if err != nil { return nil, fmt.Errorf("generate GUID failed: %w", err) } cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")) if err != nil { return nil, err } t := &winTun{ Device: deviceName, vpnNetworks: vpnNetworks, MTU: getMTU(l, c), guid: *guid, networkCategory: cat, setCategory: setCat, bypassWDF: c.GetBool("tun.windows_bypass_wdf", true), l: l, } err = t.reload(c, true, nil) if err != nil { return nil, err } var tunDevice wintun.Device tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) if err != nil { // Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device. // Trying a second time resolves the issue. l.Debug("Failed to create wintun device, retrying", "error", err) tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) if err != nil { return nil, &NameError{ Name: deviceName, Underlying: fmt.Errorf("create TUN device failed: %w", err), } } } t.tun = tunDevice.(*wintun.NativeTun) // Disable registration before Activate assigns addresses and triggers it. disableDNSRegistration(l, *guid) c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { r.RestartRequiredIfChanged("tun.dev", "tun.mtu") // Compare with the running category, not the last config. A restart with one we can't parse would fail, so say // that now rather than ask for one, and going back to the running one after a bad reload needs no restart if cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")); err != nil { t.l.Error("Invalid tun.network_category, a restart with it would fail", "error", err) r.Failed() } else if cat != t.networkCategory || setCat != t.setCategory { r.RestartRequiredIfChanged("tun.network_category") } r.RestartRequiredIfChanged(interfaceBypassKeys...) err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) r.Failed() } }) return t, nil } func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } if !initial && !change { return nil } routeTree, err := makeRouteTree(t.l, routes, false) if err != nil { return err } // Teach nebula how to handle the routes before establishing them in the system table oldRoutes := t.Routes.Swap(&routes) t.routeTree.Store(routeTree) if !initial { // Remove first, if the system removes a wanted route hopefully it will be re-added next err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes), r) if err != nil { util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) r.Failed() } // Ensure any routes we actually want are installed err = t.addRoutes(true, r) if err != nil { // Catch any stray logs util.LogWithContextIfNeeded("Failed to add routes", err, t.l) r.Failed() } } return nil } func (t *winTun) Activate() error { luid := winipcfg.LUID(t.tun.LUID()) err := luid.SetIPAddresses(t.vpnNetworks) if err != nil { return fmt.Errorf("failed to set address: %w", err) } err = t.addRoutes(false, nil) if err != nil { return err } if t.setCategory { // The wintun adapter takes a moment to register with the Network List // Manager, so we apply the category in the background and retry until // it shows up. go applyNetworkCategory(t.l, t.guid, t.networkCategory) } if t.bypassWDF { t.wdfBypass = installInterfaceBypass(t.l, uint64(t.tun.LUID())) } return nil } func (t *winTun) addRoutes(logErrors bool, rl *config.Reload) error { luid := winipcfg.LUID(t.tun.LUID()) routes := *t.Routes.Load() foundDefault4 := false carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() }) // A CIDR listed twice routes by its last entry, install only that one or each reload would flip between them. The // same rule as systemRoutes last := make(map[netip.Prefix]int, len(routes)) for i, r := range routes { last[r.Cidr] = i } for i, r := range routes { if len(r.Via) == 0 || !r.Install { // We don't allow route MTUs so only install routes with a via continue } if last[r.Cidr] != i { continue } // A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter. carriesV6 = carriesV6 || r.Cidr.Addr().Is6() // Add our unsafe route as an on-link route to the nebula tun device. result, err := addRoute(luid, r) if err != nil { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) rl.Failed() continue } else { return retErr } } else { switch result { case routeAdded: t.l.Info("Added route", "route", r) case routeMetricUpdated: t.l.Info("Updated route metric", "route", r) } } if !foundDefault4 { if r.Cidr.Bits() == 0 && r.Cidr.Addr().BitLen() == 32 { foundDefault4 = true } } } return t.setMTU(luid, foundDefault4, carriesV6) } // setMTU applies tun.mtu per address family. The default route metric rides along on the v4 handle. func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error { ipif, err := luid.IPInterface(windows.AF_INET) if err != nil { return fmt.Errorf("failed to get ip interface: %w", err) } ipif.NLMTU = uint32(t.MTU) if foundDefault4 { ipif.UseAutomaticMetric = false ipif.Metric = 0 } if err := ipif.Set(); err != nil { return fmt.Errorf("failed to set ip interface: %w", err) } // Windows tracks NLMTU per family and wintun sets neither, so v6 keeps the adapter default of 65535. // Gated so a v4-only overlay under 1280 boots; a v6 one deliberately does not, as linux also refuses. if !carriesV6 { return nil } ipif6, err := luid.IPInterface(windows.AF_INET6) if err != nil { // No v6 on the adapter means there is no NLMTU to get wrong. A failed Set below is not the same thing. t.l.Info("Skipping ipv6 MTU, no ipv6 interface on this adapter", "error", err) return nil } ipif6.NLMTU = uint32(t.MTU) if err := ipif6.Set(); err != nil { return fmt.Errorf("failed to set ipv6 interface: %w", err) } return nil } func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error { luid := winipcfg.LUID(t.tun.LUID()) for _, r := range routes { if !r.Install { continue } // See comment on luid.AddRoute err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr)) if errors.Is(err, windows.ERROR_NOT_FOUND) { // Already gone, which is what we wanted t.l.Info("Route was already removed", "route", r) } else if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) rl.Failed() } else { t.l.Info("Removed route", "route", r) } } return nil } func (t *winTun) RoutesFor(ip netip.Addr) routing.Gateways { r, _ := t.routeTree.Load().Lookup(ip) return r } func (t *winTun) Networks() []netip.Prefix { return t.vpnNetworks } func (t *winTun) Name() string { return t.Device } func (t *winTun) Write(b []byte) (int, error) { return t.tun.Write(b, 0) } func (t *winTun) Queues(int) ([]tio.Queue, error) { return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil } func (t *winTun) Close() error { // It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes, // so to be certain, just remove everything before destroying. luid := winipcfg.LUID(t.tun.LUID()) _ = luid.FlushRoutes(windows.AF_INET) _ = luid.FlushIPAddresses(windows.AF_INET) _ = luid.FlushRoutes(windows.AF_INET6) _ = luid.FlushIPAddresses(windows.AF_INET6) _ = luid.FlushDNS(windows.AF_INET) _ = luid.FlushDNS(windows.AF_INET6) if t.wdfBypass != nil { t.wdfBypass.Close() t.wdfBypass = nil } return t.tun.Close() } // systemRoutes maps each CIDR to the route that puts it in the system table. A CIDR's last entry decides, including // one that says not to install it func systemRoutes(routes []Route) map[netip.Prefix]Route { out := make(map[netip.Prefix]Route, len(routes)) for _, r := range routes { if len(r.Via) > 0 && r.Install { out[r.Cidr] = r } else { delete(out, r.Cidr) } } return out } // findRemovedSystemRoutes is the routes to delete from the system table. Windows deletes by destination and next hop, // so only a CIDR nothing installs anymore goes, a changed metric is set on the route in place by addRoute func findRemovedSystemRoutes(newRoutes, oldRoutes []Route) []Route { keep := systemRoutes(newRoutes) var removed []Route for _, r := range systemRoutes(oldRoutes) { if _, ok := keep[r.Cidr]; !ok { removed = append(removed, r) } } return removed } type routeResult int const ( routeAdded routeResult = iota routeMetricUpdated routeAlreadyInstalled ) // addRoute installs r on the interface. A reload re-adds every route to catch any the system dropped, and Windows // refuses one it has with the same destination and next hop whatever its metric. One that is there with another // metric, ours from before a metric change or someone else's, is brought to ours in place func addRoute(luid winipcfg.LUID, r Route) (routeResult, error) { nextHop := unspecifiedNextHop(r.Cidr) err := luid.AddRoute(r.Cidr, nextHop, uint32(r.Metric)) if !errors.Is(err, windows.ERROR_OBJECT_ALREADY_EXISTS) { return routeAdded, err } row, err := luid.Route(r.Cidr, nextHop) if err != nil { return routeAlreadyInstalled, err } if row.Metric != uint32(r.Metric) { row.Metric = uint32(r.Metric) return routeMetricUpdated, row.Set() } return routeAlreadyInstalled, nil } func unspecifiedNextHop(p netip.Prefix) netip.Addr { if p.Addr().Is4() { return netip.IPv4Unspecified() } return netip.IPv6Unspecified() } func generateGUIDByDeviceName(name string) (*windows.GUID, error) { // GUID is 128 bit hash := crypto.MD5.New() _, err := hash.Write([]byte(tunGUIDLabel)) if err != nil { return nil, err } _, err = hash.Write([]byte(name)) if err != nil { return nil, err } sum := hash.Sum(nil) return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil } func checkWinTunExists() error { myPath, err := os.Executable() if err != nil { return err } arch := runtime.GOARCH switch arch { case "386": //NOTE: wintun bundles 386 as x86 arch = "x86" } _, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll")) return err }