nebula/overlay/tun_darwin_route_test.go

//go:build darwin && !ios && !e2e_testing

package overlay

import (
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"net"
	"net/netip"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/slackhq/nebula/config"
	"github.com/slackhq/nebula/routing"
	"github.com/slackhq/nebula/test"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	netroute "golang.org/x/net/route"
	"golang.org/x/sys/unix"
)

// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
	msgs, err := fetchRoutes()
	require.NoError(t, err)
	routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }

	lo0, err := net.InterfaceByName("lo0")
	require.NoError(t, err)

	assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
	assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
	assert.Equal(t, lo0.Index, routeInterface("::1/128"))
	assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")

	// Most macs also list the default route scoped to their other interfaces
	out, err := exec.Command("route", "-n", "get", "default").Output()
	require.NoError(t, err)
	var name string
	for _, line := range strings.Split(string(out), "\n") {
		if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
			name = v
		}
	}
	if name == "" {
		t.Skip("no default route")
	}
	def, err := net.InterfaceByName(name)
	require.NoError(t, err)
	assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}

func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
	route := func(index, flags int) netroute.Message {
		addrs := make([]netroute.Addr, unix.RTAX_MAX)
		addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
		addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
		return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
	}
	p := netip.MustParsePrefix("192.168.1.0/24")
	assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
	assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}

// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
	tn := &tun{l: test.NewLogger()}
	routes := []Route{{
		Cidr:    netip.MustParsePrefix("10.253.1.0/24"),
		Via:     routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
		Install: true,
	}}
	tn.Routes.Store(&routes)

	assert.NoError(t, tn.addRoutes(true, nil))
	assert.NoError(t, tn.removeRoutes(routes, nil))
}

// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
	if os.Geteuid() != 0 {
		t.Skip("creating a utun and adding routes needs root")
	}
	lo0, err := net.InterfaceByName("lo0")
	require.NoError(t, err)
	cidr := "10.250.0.128/25"
	require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
	t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })

	l := test.NewLogger()
	c := config.NewC(l)
	require.NoError(t, c.LoadString("tun:\n  routes:\n    - route: "+cidr+"\n      mtu: 1300\n"))
	tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
	require.NoError(t, err)
	t.Cleanup(func() { _ = tn.Close() })
	require.NoError(t, tn.Activate())

	res := c.ReloadConfigString("tun: {}\n")
	require.NoError(t, res.Err)
	assert.False(t, res.Failed, "removing a route we never installed failed the reload")
	msgs, err := fetchRoutes()
	require.NoError(t, err)
	assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}

// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
const (
	// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
	kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
	addressRemoved     = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
	// With no interface address, which x/net/route skips
	interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
)

func TestRoutesMayHaveMoved(t *testing.T) {
	cidr := netip.MustParsePrefix("192.168.1.0/24")
	via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
	routes := []Route{{Cidr: cidr, Via: via, Install: true}}
	// A request like route(8) sends, the kernel echoes it to every routing socket
	request := func(typ, flags int, dst, mask [4]byte) []byte {
		m := &netroute.RouteMessage{
			Version: unix.RTM_VERSION,
			Type:    typ,
			Flags:   flags,
			ID:      1234,
			Addrs: []netroute.Addr{
				unix.RTAX_DST:     &netroute.Inet4Addr{IP: dst},
				unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
				unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
			},
		}
		b, err := m.Marshal()
		require.NoError(t, err)
		return b
	}
	raw := func(s string) []byte {
		b, err := hex.DecodeString(s)
		require.NoError(t, err)
		return b
	}
	lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
	routesMayHaveMoved := func(b []byte, routes []Route) bool {
		moved, _, _ := routesMayHaveMoved(b, routes)
		return moved
	}

	assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
	assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
	assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
	assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
	assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")

	failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
	binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
	assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")

	_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
	require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
	assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
	neighbor := raw(kernelSubnetDelete)
	binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
	assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")

	assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
	assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
}

// A request names its prefix and who made it, the kernel's own and interface changes name neither
func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
	cidr := netip.MustParsePrefix("192.168.1.0/24")
	routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
	m := &netroute.RouteMessage{
		Version: unix.RTM_VERSION,
		Type:    unix.RTM_CHANGE,
		Flags:   unix.RTF_UP,
		ID:      1234,
		Addrs: []netroute.Addr{
			unix.RTAX_DST:     &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
			unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
		},
	}
	b, err := m.Marshal()
	require.NoError(t, err)
	moved, p, pid := routesMayHaveMoved(b, routes)
	assert.True(t, moved)
	assert.Equal(t, cidr, p)
	assert.Equal(t, int32(1234), pid)

	for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
		b, err := hex.DecodeString(s)
		require.NoError(t, err)
		moved, p, pid := routesMayHaveMoved(b, routes)
		assert.True(t, moved, name)
		assert.False(t, p.IsValid(), name)
		assert.Zero(t, pid, name)
	}
}

func TestProcessName(t *testing.T) {
	assert.Equal(t, "launchd", processName(1))
	self := filepath.Base(os.Args[0])
	assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
	assert.Empty(t, processName(-1))
}

// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
// the minute between sweeps when it isn't zero.
func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
	requireRoot(t)

	cfg := "tun:\n  unsafe_routes:\n"
	for _, cidr := range cidrs {
		via := "10.250.0.2"
		if netip.MustParsePrefix(cidr).Addr().Is6() {
			via = "fd00:250::2"
		}
		cfg += "    - route: " + cidr + "\n      via: " + via + "\n"
	}

	l := test.NewLogger()
	c := config.NewC(l)
	require.NoError(t, c.LoadString(cfg))
	tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
	require.NoError(t, err)
	if sweep != 0 {
		tn.routeSweepEvery = sweep
	}
	t.Cleanup(func() { _ = tn.Close() })
	require.NoError(t, tn.Activate())
	return tn, c, cfg
}

func requireRoot(t *testing.T) {
	if os.Geteuid() != 0 {
		t.Skip("creating interfaces and adding routes needs root")
	}
}

// routeVia is the index of the interface that routes cidr, 0 if none does
func routeVia(t *testing.T, cidr string) int {
	msgs, err := fetchRoutes()
	require.NoError(t, err)
	return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
}

// routeStateOf is what tn made of cidr when it last looked
func routeStateOf(tn *tun, cidr string) routeState {
	tn.routesMu.Lock()
	defer tn.routesMu.Unlock()
	return tn.routeStates[netip.MustParsePrefix(cidr)]
}

func routeArgs(action, cidr string, extra ...string) []string {
	args := []string{"-q", "-n", action}
	if netip.MustParsePrefix(cidr).Addr().Is6() {
		args = append(args, "-inet6")
	}
	return append(append(args, "-net", cidr), extra...)
}

func routeCmd(t *testing.T, action, cidr string, extra ...string) {
	requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
}

// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
// says so
func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
	out, err := cmd.CombinedOutput()
	require.NoError(t, err, string(out))
	require.NotContains(t, string(out), "writing to routing socket")
}

// cleanupRoute deletes cidr's route when the test ends, whoever holds it
func cleanupRoute(t *testing.T, cidr string) {
	t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
}

// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
func routeThroughLo0(t *testing.T, cidr string) int {
	requireRoot(t)
	routeCmd(t, "add", cidr, "-interface", "lo0")
	cleanupRoute(t, cidr)
	lo0, err := net.InterfaceByName("lo0")
	require.NoError(t, err)
	return lo0.Index
}

// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
	for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
		t.Run(cidr, func(t *testing.T) {
			lo0 := routeThroughLo0(t, cidr)
			tn, _, _ := newRouteTestTun(t, 0, cidr)
			require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")

			routeCmd(t, "delete", cidr)
			assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
		})
	}
}

// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
func TestTunReloadLeavesRouteItYielded(t *testing.T) {
	cidr := "10.251.1.0/24"
	lo0 := routeThroughLo0(t, cidr)
	_, c, _ := newRouteTestTun(t, 0, cidr)

	res := c.ReloadConfigString("tun: {}\n")
	require.NoError(t, res.Err)
	assert.False(t, res.Failed)
	assert.Equal(t, lo0, routeVia(t, cidr))
}

// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
	cidr := "10.251.3.0/24"
	tn, _, _ := newRouteTestTun(t, 0, cidr)
	ours := tn.linkAddr.Load().Index
	require.Equal(t, ours, routeVia(t, cidr))

	cleanupRoute(t, cidr)
	requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
	lo0, err := net.InterfaceByName("lo0")
	require.NoError(t, err)
	time.Sleep(2 * routeRetryDelay)
	require.Equal(t, lo0.Index, routeVia(t, cidr))

	routeCmd(t, "delete", cidr)
	assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
}

// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
func TestTunLeavesRouteDeletedByHand(t *testing.T) {
	cidr := "10.251.4.0/24"
	tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
	ours := tn.linkAddr.Load().Index
	require.Equal(t, ours, routeVia(t, cidr))

	routeCmd(t, "delete", cidr)
	assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)

	// A reload with nothing changed is how to ask for it back
	res := c.ReloadConfigString(cfg)
	require.NoError(t, res.Err)
	assert.False(t, res.Failed)
	assert.Equal(t, ours, routeVia(t, cidr))
}

// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
	for name, tc := range map[string]struct {
		cidr    string
		sweep   time.Duration
		misfile func(t *testing.T, tn *tun, cidr string)
	}{
		// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
		"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
			require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
		}},
		// The sweep that caps a table that never settles
		"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
			tn.reconcileRoutes()
			require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
		}},
	} {
		t.Run(name, func(t *testing.T) {
			tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
			ours := tn.linkAddr.Load().Index
			require.Equal(t, ours, routeVia(t, tc.cidr))

			routeCmd(t, "delete", tc.cidr)
			tc.misfile(t, tn, tc.cidr)

			lo0 := routeThroughLo0(t, tc.cidr)
			require.Eventually(t, func() bool {
				s := routeStateOf(tn, tc.cidr)
				return s.kind == routeYielded && s.holder == lo0
			}, 5*time.Second, 50*time.Millisecond)

			routeCmd(t, "delete", tc.cidr)
			assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
		})
	}
}

// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
// taken for a hand delete
func TestTunYieldsToARouteChangedAway(t *testing.T) {
	for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
		t.Run(cidr, func(t *testing.T) {
			tn, _, _ := newRouteTestTun(t, 0, cidr)
			ours := tn.linkAddr.Load().Index
			require.Equal(t, ours, routeVia(t, cidr))

			cleanupRoute(t, cidr)
			routeCmd(t, "change", cidr, "-interface", "lo0")
			lo0, err := net.InterfaceByName("lo0")
			require.NoError(t, err)
			require.Equal(t, lo0.Index, routeVia(t, cidr))
			require.Eventually(t, func() bool {
				s := routeStateOf(tn, cidr)
				return s.kind == routeYielded && s.holder == lo0.Index
			}, 5*time.Second, 50*time.Millisecond)

			routeCmd(t, "delete", cidr)
			assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
		})
	}
}

// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
// ethernet configd routed the lan through
func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
	requireRoot(t)
	var feth string
	for i := 100; i < 200 && feth == ""; i++ {
		if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
			feth = fmt.Sprintf("feth%d", i)
		}
	}
	require.NotEmpty(t, feth)
	require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
	t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
	require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())

	// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
	// refuses one named by the feth itself
	cidr := "10.251.2.0/24"
	routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
	cleanupRoute(t, cidr)
	ifc, err := net.InterfaceByName(feth)
	require.NoError(t, err)

	tn, _, _ := newRouteTestTun(t, 0, cidr)
	require.Equal(t, ifc.Index, routeVia(t, cidr))

	// Losing the address first leaves the route, the reconcile finds it still held
	require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
	time.Sleep(2 * routeRetryDelay)
	require.Equal(t, ifc.Index, routeVia(t, cidr))

	// Only RTM_IFINFO says the interface went, its route goes with no message of its own
	require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
	assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}

// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
	cidr := "10.251.5.0/24"
	routeThroughLo0(t, cidr)
	tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)

	tn.routesMu.Lock()
	require.NoError(t, tn.routeWatch.Close())
	tn.routesMu.Unlock()

	routeCmd(t, "delete", cidr)
	assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
}

// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
func TestTunDevNeedsRestart(t *testing.T) {
	for name, want := range map[string]bool{
		"":      false,
		"utun":  false,
		"nope":  false,
		"utun7": false,
		"utun3": true,
	} {
		assert.Equal(t, want, tunDevNeedsRestart(name, "utun7"), name)
	}
}