use a trampoline to get to the spicy _x functions

internal/msgx/doc.go

@@ -0,0 +1,5 @@
+// Package msgx calls xnu's private batched socket calls, sendmsg_x and recvmsg_x, through libSystem.
+//
+// The calls are not public API, so the package never links them: it looks them up with dlsym on first use.
+// The package is darwin-only, and on iOS Available is always false, so App Store builds keep to public API.
+package msgx

internal/msgx/hdr_darwin.go

@@ -0,0 +1,16 @@
+package msgx
+
+import "golang.org/x/sys/unix"
+
+// Hdr mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h), which the SDK doesn't ship.
+// recvmsg_x reports each message's length in Datalen rather than in its return value.
+type Hdr struct {
+	Name       *byte
+	Namelen    uint32
+	Iov        *unix.Iovec
+	Iovlen     int32
+	Control    *byte
+	Controllen uint32
+	Flags      int32
+	Datalen    uint64
+}

internal/msgx/msgx_darwin.go

@@ -0,0 +1,77 @@
+//go:build !ios
+
+package msgx
+
+import (
+	"sync"
+	"syscall"
+	"unsafe"
+)
+
+// dlsym is public libSystem API, imported the way x/sys/unix imports its libSystem stubs. The local name is
+// distinct from runtime/race's dlsym import.
+//
+//go:cgo_import_dynamic msgx_dlsym dlsym "/usr/lib/libSystem.B.dylib"
+
+// dlsymTrampolineAddr is the address of msgx_darwin.s's jump to dlsym.
+var dlsymTrampolineAddr uintptr
+
+// syscall6X calls the C function at fn on the system stack and returns errno when it returns a 64-bit -1.
+// It is how x/sys/unix calls libSystem, and the runtime keeps it linkable for that.
+//
+//go:linkname syscall6X syscall.syscall6X
+func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno)
+
+// rtldDefault is dlfcn.h's RTLD_DEFAULT, ((void *)-2): search every image loaded in the process.
+const rtldDefault = ^uintptr(1)
+
+var (
+	sendmsgXName = []byte("sendmsg_x\x00")
+	recvmsgXName = []byte("recvmsg_x\x00")
+)
+
+type funcs struct {
+	sendmsgX uintptr
+	recvmsgX uintptr
+}
+
+var resolve = sync.OnceValue(func() funcs {
+	return funcs{sendmsgX: lookup(sendmsgXName), recvmsgX: lookup(recvmsgXName)}
+})
+
+// lookup returns the address of the NUL-terminated C symbol name, or 0 when no loaded image exports it.
+func lookup(name []byte) uintptr {
+	p, _, _ := syscall6X(dlsymTrampolineAddr, rtldDefault, uintptr(unsafe.Pointer(&name[0])), 0, 0, 0, 0)
+	return p
+}
+
+// Available reports whether libSystem exports both sendmsg_x and recvmsg_x. When it is false, Send and Recv
+// return ENOSYS.
+func Available() bool {
+	f := resolve()
+	return f.sendmsgX != 0 && f.recvmsgX != 0
+}
+
+// Send passes hdrs to sendmsg_x on fd and returns how many messages the kernel took.
+func Send(fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) {
+	return call(resolve().sendmsgX, fd, hdrs, flags)
+}
+
+// Recv passes hdrs to recvmsg_x on fd and returns how many messages it filled.
+func Recv(fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) {
+	return call(resolve().recvmsgX, fd, hdrs, flags)
+}
+
+func call(fn, fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) {
+	if fn == 0 {
+		return 0, syscall.ENOSYS
+	}
+	if len(hdrs) == 0 {
+		return 0, syscall.EINVAL
+	}
+	r, _, errno := syscall6X(fn, fd, uintptr(unsafe.Pointer(&hdrs[0])), uintptr(len(hdrs)), uintptr(flags), 0, 0)
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(r), 0
+}

internal/msgx/msgx_darwin.s

@@ -0,0 +1,9 @@
+//go:build !ios
+
+#include "textflag.h"
+
+// The same trampoline shape x/sys/unix uses for its libSystem stubs; it assembles for amd64 and arm64 alike.
+TEXT msgx_dlsym_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	msgx_dlsym(SB)
+GLOBL	·dlsymTrampolineAddr(SB), RODATA, $8
+DATA	·dlsymTrampolineAddr(SB)/8, $msgx_dlsym_trampoline<>(SB)

internal/msgx/msgx_darwin_test.go

@@ -0,0 +1,69 @@
+//go:build !ios
+
+package msgx
+
+import (
+	"bytes"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// TestAvailable pins that dlsym finds both calls in this macOS's libSystem, so the batched paths are live.
+func TestAvailable(t *testing.T) {
+	assert.True(t, Available())
+}
+
+// TestSendRecvRoundTrip pins the libSystem calling path end to end: a batch goes out through sendmsg_x in one
+// call and comes back through recvmsg_x in one call, each message whole, in order, with Datalen set.
+func TestSendRecvRoundTrip(t *testing.T) {
+	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = unix.Close(fds[0])
+		_ = unix.Close(fds[1])
+	})
+	// A dropped datagram fails the receive instead of hanging it.
+	require.NoError(t, unix.SetsockoptTimeval(fds[1], unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Sec: 2}))
+
+	const n = 5
+	var want [n][]byte
+	var iovs [n]unix.Iovec
+	hdrs := make([]Hdr, n)
+	for i := range n {
+		want[i] = bytes.Repeat([]byte{byte(i + 1)}, 10+i)
+		iovs[i] = unix.Iovec{Base: &want[i][0]}
+		iovs[i].SetLen(len(want[i]))
+		hdrs[i] = Hdr{Iov: &iovs[i], Iovlen: 1}
+	}
+	sent, errno := Send(uintptr(fds[0]), hdrs, 0)
+	require.Zero(t, errno)
+	require.Equal(t, n, sent)
+
+	var bufs [n + 1][64]byte
+	var riovs [n + 1]unix.Iovec
+	rhdrs := make([]Hdr, n+1)
+	for i := range rhdrs {
+		riovs[i] = unix.Iovec{Base: &bufs[i][0]}
+		riovs[i].SetLen(len(bufs[i]))
+		rhdrs[i] = Hdr{Iov: &riovs[i], Iovlen: 1}
+	}
+	got, errno := Recv(uintptr(fds[1]), rhdrs, 0)
+	require.Zero(t, errno)
+	require.Equal(t, n, got)
+	for i := range n {
+		l := int(rhdrs[i].Datalen)
+		assert.Equal(t, want[i], bufs[i][:l], "message %d", i)
+	}
+}
+
+// TestEmptyBatch pins that an empty batch is refused before it reaches the kernel.
+func TestEmptyBatch(t *testing.T) {
+	_, errno := Send(0, nil, 0)
+	assert.Equal(t, syscall.EINVAL, errno)
+	_, errno = Recv(0, nil, 0)
+	assert.Equal(t, syscall.EINVAL, errno)
+}

internal/msgx/msgx_ios.go

@@ -0,0 +1,12 @@
+package msgx
+
+import "syscall"
+
+// Available is always false on iOS, so App Store builds never look up private symbols.
+func Available() bool { return false }
+
+// Send always returns ENOSYS on iOS.
+func Send(uintptr, []Hdr, int) (int, syscall.Errno) { return 0, syscall.ENOSYS }
+
+// Recv always returns ENOSYS on iOS.
+func Recv(uintptr, []Hdr, int) (int, syscall.Errno) { return 0, syscall.ENOSYS }

overlay/tio/tio_utun_darwin.go

@@ -9,8 +9,9 @@ import (
 	"sync"
 	"sync/atomic"
 	"syscall"
-	"unsafe"
+	_ "unsafe" // for go:linkname
 
+	"github.com/slackhq/nebula/internal/msgx"
 	"golang.org/x/sys/unix"
 )
 
@@ -40,7 +41,7 @@ type Utun struct {
 	readBuf []byte
 	readRet [tunReadBatch]Packet
 
-	// noSendmsgX is set once sendmsg_x is refused; WriteBatch then writes one packet at a time.
+	// noSendmsgX is set when libSystem lacks sendmsg_x or the kernel refuses it; WriteBatch then writes one packet at a time.
 	noSendmsgX atomic.Bool
 	batchMu    sync.Mutex
 	batch      tunBatch
@@ -50,23 +51,10 @@ type Utun struct {
 type tunBatch struct {
 	heads [tunWriteBatch][4]byte
 	iovs  [tunWriteBatch][2]unix.Iovec
-	hdrs  [tunWriteBatch]msghdrX
+	hdrs  [tunWriteBatch]msgx.Hdr
 	pkts  [tunWriteBatch][]byte
 }
 
-// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h).
-// Keep in sync with msghdrX in udp/udp_darwin.go.
-type msghdrX struct {
-	Name       *byte
-	Namelen    uint32
-	Iov        *unix.Iovec
-	Iovlen     int32
-	Control    *byte
-	Controllen uint32
-	Flags      int32
-	Datalen    uint64
-}
-
 // NewUtun wraps a utun control socket fd and makes it non-blocking.
 // On failure it does NOT close fd: the caller owns fd until NewUtun succeeds, after which Close closes it.
 func NewUtun(fd int, l *slog.Logger) (*Utun, error) {
@@ -74,11 +62,13 @@ func NewUtun(fd int, l *slog.Logger) (*Utun, error) {
 	if err := unix.SetNonblock(fd, true); err != nil {
 		return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
 	}
-	return &Utun{
+	u := &Utun{
 		f:       os.NewFile(uintptr(fd), "utun"),
 		l:       l,
 		readBuf: make([]byte, tunReadArena),
-	}, nil
+	}
+	u.noSendmsgX.Store(!msgx.Available())
+	return u, nil
 }
 
 // tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the
@@ -246,7 +236,7 @@ func (u *Utun) WriteBatch(pkts [][]byte) error {
 				{Base: &b.heads[n][0], Len: 4},
 				{Base: &p[0], Len: uint64(len(p))},
 			}
-			b.hdrs[n] = msghdrX{Iov: &b.iovs[n][0], Iovlen: 2}
+			b.hdrs[n] = msgx.Hdr{Iov: &b.iovs[n][0], Iovlen: 2}
 			b.pkts[n] = p
 			n++
 		}
@@ -306,10 +296,10 @@ func (u *Utun) sendmsgX(off, n int) (int, error) {
 	if err != nil {
 		return 0, err
 	}
-	var sent uintptr
+	var sent int
 	var errno syscall.Errno
 	err = rc.Write(func(fd uintptr) bool {
-		sent, _, errno = unix.Syscall6(unix.SYS_SENDMSG_X, fd, uintptr(unsafe.Pointer(&u.batch.hdrs[off])), uintptr(n-off), 0, 0, 0)
+		sent, errno = msgx.Send(fd, u.batch.hdrs[off:n], 0)
 		// sendmsg_x reports EAGAIN only when it took nothing; a partial batch returns its count.
 		return !errno.Temporary()
 	})
@@ -319,7 +309,7 @@ func (u *Utun) sendmsgX(off, n int) (int, error) {
 	if errno != 0 {
 		return 0, errno
 	}
-	return int(sent), nil
+	return sent, nil
 }
 
 // writeEach writes pkts one at a time with Write.

udp/udp_darwin.go

@@ -15,6 +15,7 @@ import (
 	"unsafe"
 
 	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/internal/msgx"
 	"golang.org/x/sys/unix"
 )
 
@@ -39,6 +40,7 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
 
 	if uc, ok := pc.(*net.UDPConn); ok {
 		c := &StdConn{UDPConn: uc, l: l}
+		c.noSendmsgX.Store(!msgx.Available())
 
 		rc, err := uc.SyscallConn()
 		if err != nil {
@@ -150,7 +152,7 @@ const sendBatch = 64
 type sendScratch struct {
 	names [sendBatch]unix.RawSockaddrInet6
 	iovs  [sendBatch]unix.Iovec
-	hdrs  [sendBatch]msghdrX
+	hdrs  [sendBatch]msgx.Hdr
 	// idx maps each entry back to its index in WriteBatch's bufs.
 	idx [sendBatch]int
 }
@@ -190,7 +192,7 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
 				b.iovs[n].Base = &bufs[i][0]
 				b.iovs[n].SetLen(len(bufs[i]))
 			}
-			b.hdrs[n] = msghdrX{Name: (*byte)(unsafe.Pointer(&b.names[n])), Namelen: namelen, Iov: &b.iovs[n], Iovlen: 1}
+			b.hdrs[n] = msgx.Hdr{Name: (*byte)(unsafe.Pointer(&b.names[n])), Namelen: namelen, Iov: &b.iovs[n], Iovlen: 1}
 			b.idx[n] = i
 			n++
 		}
@@ -252,11 +254,11 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
 // The socket is non-blocking, so a full send buffer returns EAGAIN rather than waiting.
 func (u *StdConn) sendmsgX(off, end int) (int, syscall.Errno) {
 	for {
-		r0, _, errno := unix.Syscall6(unix.SYS_SENDMSG_X, u.sysFd, uintptr(unsafe.Pointer(&u.sb.hdrs[off])), uintptr(end-off), 0, 0, 0)
+		sent, errno := msgx.Send(u.sysFd, u.sb.hdrs[off:end], 0)
 		if errno == unix.EINTR {
 			continue
 		}
-		return int(r0), errno
+		return sent, errno
 	}
 }
 
@@ -330,23 +332,12 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
 // recvBatch is how many datagrams one recvmsg_x call may return.
 const recvBatch = 64
 
-// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h). recvmsg_x reports each
-// datagram's length in Datalen rather than in its return value.
-// Keep in sync with msghdrX in overlay/tio/tio_utun_darwin.go.
-type msghdrX struct {
-	Name       *byte
-	Namelen    uint32
-	Iov        *unix.Iovec
-	Iovlen     int32
-	Control    *byte
-	Controllen uint32
-	Flags      int32
-	Datalen    uint64
-}
-
 // ListenOut drains the socket with recvmsg_x, darwin's private batched recvmsg, so one syscall reads
 // up to recvBatch datagrams instead of one.
 func (u *StdConn) ListenOut(r EncReader, flush func()) error {
+	if !msgx.Available() {
+		return u.listenOutSingle(r, flush)
+	}
 	rc, err := u.UDPConn.SyscallConn()
 	if err != nil {
 		return err
@@ -355,7 +346,7 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
 	bufs := make([][]byte, recvBatch)
 	names := make([]unix.RawSockaddrInet6, recvBatch)
 	iovs := make([]unix.Iovec, recvBatch)
-	hdrs := make([]msghdrX, recvBatch)
+	hdrs := make([]msgx.Hdr, recvBatch)
 	for i := range hdrs {
 		bufs[i] = make([]byte, MTU)
 		iovs[i].Base = &bufs[i][0]
@@ -374,12 +365,8 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
 				hdrs[i].Flags = 0
 				hdrs[i].Datalen = 0
 			}
-			r0, _, e := unix.Syscall6(unix.SYS_RECVMSG_X, fd, uintptr(unsafe.Pointer(&hdrs[0])), recvBatch, 0, 0, 0)
-			if e == unix.EAGAIN {
-				return false
-			}
-			n, errno = int(r0), e
-			return true
+			n, errno = msgx.Recv(fd, hdrs, 0)
+			return errno != unix.EAGAIN
 		})
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
@@ -389,7 +376,7 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
 			continue
 		}
 		if errno == unix.ENOSYS || errno == unix.EPERM || errno == unix.EOPNOTSUPP {
-			// recvmsg_x is private, so a kernel or sandbox (such as an iOS extension's) may refuse it.
+			// recvmsg_x is private, so a kernel or sandbox may refuse it.
 			u.l.Warn("recvmsg_x unavailable, reading one datagram per syscall", "error", errno)
 			return u.listenOutSingle(r, flush)
 		}
@@ -419,7 +406,7 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
 
 // checkMsghdrX rejects a recvmsg_x result that intact msghdr_x entries can't produce, which is how a change
 // to xnu's private struct layout would show up.
-func checkMsghdrX(hdrs []msghdrX, names []unix.RawSockaddrInet6, n int) error {
+func checkMsghdrX(hdrs []msgx.Hdr, names []unix.RawSockaddrInet6, n int) error {
 	if n > len(hdrs) {
 		return fmt.Errorf("returned %d datagrams for %d headers", n, len(hdrs))
 	}

udp/udp_recvmsgx_darwin_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/slackhq/nebula/internal/msgx"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/unix"
@@ -81,7 +82,7 @@ func TestListenOutBatchesBurst(t *testing.T) {
 	}
 }
 
-// msghdrX round-trips through xnu's recvmsg_x on a loopback socket, v4 and v6. The kernel reads Name, Namelen,
+// msgx.Hdr round-trips through xnu's recvmsg_x on a loopback socket, v4 and v6. The kernel reads Name, Namelen,
 // Iov and Iovlen from each entry and writes Namelen, Flags and Datalen back, so a field at the wrong offset or a
 // wrong stride between entries shows up as a wrong length, sender, or payload.
 // TestListenOutSingleFallback pins the path ListenOut takes when the kernel refuses recvmsg_x.
@@ -134,8 +135,8 @@ func TestListenOutSingleFallback(t *testing.T) {
 }
 
 func TestCheckMsghdrX(t *testing.T) {
-	valid := func() ([]msghdrX, []unix.RawSockaddrInet6) {
-		hdrs := make([]msghdrX, 2)
+	valid := func() ([]msgx.Hdr, []unix.RawSockaddrInet6) {
+		hdrs := make([]msgx.Hdr, 2)
 		names := make([]unix.RawSockaddrInet6, 2)
 		hdrs[0].Namelen, hdrs[0].Datalen = unix.SizeofSockaddrInet4, 1
 		names[0].Family = unix.AF_INET
@@ -151,12 +152,12 @@ func TestCheckMsghdrX(t *testing.T) {
 	for _, tc := range []struct {
 		name   string
 		n      int
-		mutate func([]msghdrX, []unix.RawSockaddrInet6)
+		mutate func([]msgx.Hdr, []unix.RawSockaddrInet6)
 	}{
-		{"count past headers", 3, func([]msghdrX, []unix.RawSockaddrInet6) {}},
-		{"datalen past buffer", 2, func(h []msghdrX, _ []unix.RawSockaddrInet6) { h[1].Datalen = MTU + 1 }},
-		{"unknown family", 2, func(_ []msghdrX, a []unix.RawSockaddrInet6) { a[0].Family = unix.AF_UNIX }},
-		{"family and length disagree", 2, func(h []msghdrX, _ []unix.RawSockaddrInet6) { h[0].Namelen = unix.SizeofSockaddrInet6 }},
+		{"count past headers", 3, func([]msgx.Hdr, []unix.RawSockaddrInet6) {}},
+		{"datalen past buffer", 2, func(h []msgx.Hdr, _ []unix.RawSockaddrInet6) { h[1].Datalen = MTU + 1 }},
+		{"unknown family", 2, func(_ []msgx.Hdr, a []unix.RawSockaddrInet6) { a[0].Family = unix.AF_UNIX }},
+		{"family and length disagree", 2, func(h []msgx.Hdr, _ []unix.RawSockaddrInet6) { h[0].Namelen = unix.SizeofSockaddrInet6 }},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			hdrs, names := valid()
@@ -238,7 +239,7 @@ func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Du
 	bufs := make([][]byte, nh)
 	names := make([]unix.RawSockaddrInet6, nh)
 	iovs := make([]unix.Iovec, nh)
-	hdrs := make([]msghdrX, nh)
+	hdrs := make([]msgx.Hdr, nh)
 	for i := range hdrs {
 		bufs[i] = make([]byte, MTU)
 		iovs[i].Base = &bufs[i][0]
@@ -253,7 +254,7 @@ func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Du
 	for next := 0; next < len(lens); {
 		for i := range hdrs {
 			clear(bufs[i])
-			hdrs[i] = msghdrX{
+			hdrs[i] = msgx.Hdr{
 				Name:    (*byte)(unsafe.Pointer(&names[i])),
 				Namelen: unix.SizeofSockaddrInet6,
 				Iov:     &iovs[i],
@@ -265,9 +266,8 @@ func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Du
 		var n int
 		var errno syscall.Errno
 		require.NoError(t, rc.Read(func(fd uintptr) bool {
-			r0, _, e := unix.Syscall6(unix.SYS_RECVMSG_X, fd, uintptr(unsafe.Pointer(&hdrs[0])), uintptr(nh), 0, 0, 0)
-			n, errno = int(r0), e
-			return e != unix.EAGAIN
+			n, errno = msgx.Recv(fd, hdrs[:nh], 0)
+			return errno != unix.EAGAIN
 		}), "received %d of %d datagrams", next, len(lens))
 		require.Zero(t, errno)
 		require.Positive(t, n)