nebula/overlay/tun_windows.go

//go:build !e2e_testing

package overlay

import (
	"crypto"
	"errors"
	"fmt"
	"log/slog"
	"net/netip"
	"os"
	"path/filepath"
	"runtime"
	"slices"
	"sync/atomic"
	"syscall"
	"unsafe"

	"github.com/gaissmai/bart"
	"github.com/slackhq/nebula/config"
	"github.com/slackhq/nebula/overlay/tio"
	"github.com/slackhq/nebula/routing"
	"github.com/slackhq/nebula/util"
	"github.com/slackhq/nebula/wintun"
	"golang.org/x/sys/windows"
	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
)

type closer interface {
	Close()
}

const tunGUIDLabel = "Fixed Nebula Windows GUID v1"

type winTun struct {
	Device          string
	vpnNetworks     []netip.Prefix
	MTU             int
	Routes          atomic.Pointer[[]Route]
	routeTree       atomic.Pointer[bart.Table[routing.Gateways]]
	guid            windows.GUID
	networkCategory networkCategory
	setCategory     bool
	bypassWDF       bool
	wdfBypass       closer
	l               *slog.Logger

	tun *wintun.NativeTun
}

func (t *winTun) Read(b []byte) (int, error) {
	return t.tun.Read(b, 0)
}

func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) {
	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
}

func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) {
	err := checkWinTunExists()
	if err != nil {
		return nil, fmt.Errorf("can not load the wintun driver: %w", err)
	}

	deviceName := c.GetString("tun.dev", "")
	guid, err := generateGUIDByDeviceName(deviceName)
	if err != nil {
		return nil, fmt.Errorf("generate GUID failed: %w", err)
	}

	cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private"))
	if err != nil {
		return nil, err
	}

	t := &winTun{
		Device:          deviceName,
		vpnNetworks:     vpnNetworks,
		MTU:             getMTU(l, c),
		guid:            *guid,
		networkCategory: cat,
		setCategory:     setCat,
		bypassWDF:       c.GetBool("tun.windows_bypass_wdf", true),
		l:               l,
	}

	err = t.reload(c, true, nil)
	if err != nil {
		return nil, err
	}

	var tunDevice wintun.Device
	tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
	if err != nil {
		// Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device.
		// Trying a second time resolves the issue.
		l.Debug("Failed to create wintun device, retrying", "error", err)
		tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
		if err != nil {
			return nil, &NameError{
				Name:       deviceName,
				Underlying: fmt.Errorf("create TUN device failed: %w", err),
			}
		}
	}
	t.tun = tunDevice.(*wintun.NativeTun)

	// Disable registration before Activate assigns addresses and triggers it.
	disableDNSRegistration(l, *guid)

	c.RegisterReloadCallback(func(c *config.C, r *config.Reload) {
		r.RestartRequiredIfChanged("tun.dev", "tun.mtu")
		// Compare with the running category, not the last config. A restart with one we can't parse would fail, so say
		// that now rather than ask for one, and going back to the running one after a bad reload needs no restart
		if cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")); err != nil {
			t.l.Error("Invalid tun.network_category, a restart with it would fail", "error", err)
			r.Failed()
		} else if cat != t.networkCategory || setCat != t.setCategory {
			r.RestartRequiredIfChanged("tun.network_category")
		}
		r.RestartRequiredIfChanged(interfaceBypassKeys...)
		err := t.reload(c, false, r)
		if err != nil {
			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
			r.Failed()
		}
	})

	return t, nil
}

func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error {
	change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial)
	if err != nil {
		return err
	}

	if !initial && !change {
		return nil
	}

	routeTree, err := makeRouteTree(t.l, routes, false)
	if err != nil {
		return err
	}

	// Teach nebula how to handle the routes before establishing them in the system table
	oldRoutes := t.Routes.Swap(&routes)
	t.routeTree.Store(routeTree)

	if !initial {
		// Remove first, if the system removes a wanted route hopefully it will be re-added next
		err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes), r)
		if err != nil {
			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
			r.Failed()
		}

		// Ensure any routes we actually want are installed
		err = t.addRoutes(true, r)
		if err != nil {
			// Catch any stray logs
			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
			r.Failed()
		}
	}

	return nil
}

func (t *winTun) Activate() error {
	luid := winipcfg.LUID(t.tun.LUID())

	err := luid.SetIPAddresses(t.vpnNetworks)
	if err != nil {
		return fmt.Errorf("failed to set address: %w", err)
	}

	err = t.addRoutes(false, nil)
	if err != nil {
		return err
	}

	if t.setCategory {
		// The wintun adapter takes a moment to register with the Network List
		// Manager, so we apply the category in the background and retry until
		// it shows up.
		go applyNetworkCategory(t.l, t.guid, t.networkCategory)
	}

	if t.bypassWDF {
		t.wdfBypass = installInterfaceBypass(t.l, uint64(t.tun.LUID()))
	}

	return nil
}

func (t *winTun) addRoutes(logErrors bool, rl *config.Reload) error {
	luid := winipcfg.LUID(t.tun.LUID())
	routes := *t.Routes.Load()
	foundDefault4 := false
	carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })

	// A CIDR listed twice routes by its last entry, install only that one or each reload would flip between them. The
	// same rule as systemRoutes
	last := make(map[netip.Prefix]int, len(routes))
	for i, r := range routes {
		last[r.Cidr] = i
	}

	for i, r := range routes {
		if len(r.Via) == 0 || !r.Install {
			// We don't allow route MTUs so only install routes with a via
			continue
		}
		if last[r.Cidr] != i {
			continue
		}

		// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
		carriesV6 = carriesV6 || r.Cidr.Addr().Is6()

		// Add our unsafe route as an on-link route to the nebula tun device.
		result, err := addRoute(luid, r)
		if err != nil {
			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
			if logErrors {
				retErr.Log(t.l)
				rl.Failed()
				continue
			} else {
				return retErr
			}
		} else {
			switch result {
			case routeAdded:
				t.l.Info("Added route", "route", r)
			case routeMetricUpdated:
				t.l.Info("Updated route metric", "route", r)
			}
		}

		if !foundDefault4 {
			if r.Cidr.Bits() == 0 && r.Cidr.Addr().BitLen() == 32 {
				foundDefault4 = true
			}
		}
	}

	return t.setMTU(luid, foundDefault4, carriesV6)
}

// setMTU applies tun.mtu per address family. The default route metric rides along on the v4 handle.
func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error {
	ipif, err := luid.IPInterface(windows.AF_INET)
	if err != nil {
		return fmt.Errorf("failed to get ip interface: %w", err)
	}

	ipif.NLMTU = uint32(t.MTU)
	if foundDefault4 {
		ipif.UseAutomaticMetric = false
		ipif.Metric = 0
	}

	if err := ipif.Set(); err != nil {
		return fmt.Errorf("failed to set ip interface: %w", err)
	}

	// Windows tracks NLMTU per family and wintun sets neither, so v6 keeps the adapter default of 65535.
	// Gated so a v4-only overlay under 1280 boots; a v6 one deliberately does not, as linux also refuses.
	if !carriesV6 {
		return nil
	}

	ipif6, err := luid.IPInterface(windows.AF_INET6)
	if err != nil {
		// No v6 on the adapter means there is no NLMTU to get wrong. A failed Set below is not the same thing.
		t.l.Info("Skipping ipv6 MTU, no ipv6 interface on this adapter", "error", err)
		return nil
	}

	ipif6.NLMTU = uint32(t.MTU)
	if err := ipif6.Set(); err != nil {
		return fmt.Errorf("failed to set ipv6 interface: %w", err)
	}

	return nil
}

func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error {
	luid := winipcfg.LUID(t.tun.LUID())

	for _, r := range routes {
		if !r.Install {
			continue
		}

		// See comment on luid.AddRoute
		err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr))
		if errors.Is(err, windows.ERROR_NOT_FOUND) {
			// Already gone, which is what we wanted
			t.l.Info("Route was already removed", "route", r)
		} else if err != nil {
			t.l.Error("Failed to remove route", "error", err, "route", r)
			rl.Failed()
		} else {
			t.l.Info("Removed route", "route", r)
		}
	}
	return nil
}

func (t *winTun) RoutesFor(ip netip.Addr) routing.Gateways {
	r, _ := t.routeTree.Load().Lookup(ip)
	return r
}

func (t *winTun) Networks() []netip.Prefix {
	return t.vpnNetworks
}

func (t *winTun) Name() string {
	return t.Device
}

func (t *winTun) Write(b []byte) (int, error) {
	return t.tun.Write(b, 0)
}

func (t *winTun) Queues(int) ([]tio.Queue, error) {
	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
}

func (t *winTun) Close() error {
	// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
	// so to be certain, just remove everything before destroying.
	luid := winipcfg.LUID(t.tun.LUID())
	_ = luid.FlushRoutes(windows.AF_INET)
	_ = luid.FlushIPAddresses(windows.AF_INET)

	_ = luid.FlushRoutes(windows.AF_INET6)
	_ = luid.FlushIPAddresses(windows.AF_INET6)

	_ = luid.FlushDNS(windows.AF_INET)
	_ = luid.FlushDNS(windows.AF_INET6)

	if t.wdfBypass != nil {
		t.wdfBypass.Close()
		t.wdfBypass = nil
	}

	return t.tun.Close()
}

// systemRoutes maps each CIDR to the route that puts it in the system table. A CIDR's last entry decides, including
// one that says not to install it
func systemRoutes(routes []Route) map[netip.Prefix]Route {
	out := make(map[netip.Prefix]Route, len(routes))
	for _, r := range routes {
		if len(r.Via) > 0 && r.Install {
			out[r.Cidr] = r
		} else {
			delete(out, r.Cidr)
		}
	}
	return out
}

// findRemovedSystemRoutes is the routes to delete from the system table. Windows deletes by destination and next hop,
// so only a CIDR nothing installs anymore goes, a changed metric is set on the route in place by addRoute
func findRemovedSystemRoutes(newRoutes, oldRoutes []Route) []Route {
	keep := systemRoutes(newRoutes)
	var removed []Route
	for _, r := range systemRoutes(oldRoutes) {
		if _, ok := keep[r.Cidr]; !ok {
			removed = append(removed, r)
		}
	}
	return removed
}

type routeResult int

const (
	routeAdded routeResult = iota
	routeMetricUpdated
	routeAlreadyInstalled
)

// addRoute installs r on the interface. A reload re-adds every route to catch any the system dropped, and Windows
// refuses one it has with the same destination and next hop whatever its metric. One that is there with another
// metric, ours from before a metric change or someone else's, is brought to ours in place
func addRoute(luid winipcfg.LUID, r Route) (routeResult, error) {
	nextHop := unspecifiedNextHop(r.Cidr)
	err := luid.AddRoute(r.Cidr, nextHop, uint32(r.Metric))
	if !errors.Is(err, windows.ERROR_OBJECT_ALREADY_EXISTS) {
		return routeAdded, err
	}
	row, err := luid.Route(r.Cidr, nextHop)
	if err != nil {
		return routeAlreadyInstalled, err
	}
	if row.Metric != uint32(r.Metric) {
		row.Metric = uint32(r.Metric)
		return routeMetricUpdated, row.Set()
	}
	return routeAlreadyInstalled, nil
}

func unspecifiedNextHop(p netip.Prefix) netip.Addr {
	if p.Addr().Is4() {
		return netip.IPv4Unspecified()
	}
	return netip.IPv6Unspecified()
}

func generateGUIDByDeviceName(name string) (*windows.GUID, error) {
	// GUID is 128 bit
	hash := crypto.MD5.New()

	_, err := hash.Write([]byte(tunGUIDLabel))
	if err != nil {
		return nil, err
	}

	_, err = hash.Write([]byte(name))
	if err != nil {
		return nil, err
	}

	sum := hash.Sum(nil)

	return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil
}

func checkWinTunExists() error {
	myPath, err := os.Executable()
	if err != nil {
		return err
	}

	arch := runtime.GOARCH
	switch arch {
	case "386":
		//NOTE: wintun bundles 386 as x86
		arch = "x86"
	}

	_, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll"))
	return err
}