tun: Allow excluding CPUs from process threads at startup

Set tun.avoid_cpus to a list of logical CPUs to remove from the affinity of
every thread of the process on Linux, before services start and reader CPUs
are chosen. Later threads inherit the restricted mask; enumeration repeats so
threads created meanwhile are covered. Inherited taskset and cgroup
restrictions are kept. Keeping Nebula off the underlay NIC's receive IRQ CPUs
and their SMT siblings is what this is for; the example configuration shows
how to find them.

Invalid IDs, a list that would leave a thread no CPU, and syscall failures
fail startup. Configuration tests only validate. Other platforms log and
ignore the setting. The exclusion is process-wide, including embedded use, and
not reloadable. It is independent of thread pinning.

New API: util.AvoidCPUs, next to the existing PinThreadToCPU and AllowedCPUs.

Tests run in a child process and check existing and new threads, preserved
inherited masks, validation without changes, and the unix.CPUSet bounds on
32- and 64-bit platforms. No isolated performance claim is made.

examples/config.yml

@@ -295,6 +295,14 @@ tun:
   #  - 2
   #  - 4
 
+  # Linux only. avoid_cpus removes these CPU IDs from the affinity of every thread of the process at startup,
+  # embedding applications included; later threads inherit the remaining set. Applied before cpu_affinity and the
+  # default pin selection, independent of pin_threads. Invalid IDs, or IDs that would leave no CPU, fail
+  # startup. Not reloadable; ignored on other platforms. To keep Nebula off the underlay NIC's receive IRQ CPUs, find
+  # them in /proc/interrupts and /proc/irq/<IRQ>/effective_affinity_list, add their SMT siblings from
+  # /sys/devices/system/cpu/cpu<N>/topology/thread_siblings_list, and revisit the list when IRQ placement changes.
+  #avoid_cpus: []
+
   # Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
   routes:
     #- mtu: 8800

main.go

@@ -2,6 +2,7 @@ package nebula
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net"
@@ -31,6 +32,10 @@ const maxRoutines = 1 << 16
 const _ = noiseutil.RejectHeadroom - 4*maxRoutines
 
 func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
+	// Apply before starting services or selecting per-reader pin CPUs.
+	if err := configureAvoidCPUs(c, l, configTest); err != nil {
+		return nil, err
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 	// The goroutines started below stop only when this context does, and only a caller holding the
 	// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
@@ -416,6 +421,41 @@ func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
 	return cpus
 }
 
+// configureAvoidCPUs applies tun.avoid_cpus, a list of CPU IDs to keep every thread of the process off. Unlike
+// tun.cpu_affinity, a bad entry fails startup rather than being ignored. Configuration tests only validate it.
+func configureAvoidCPUs(c *config.C, l *slog.Logger, configTest bool) error {
+	raw := c.Get("tun.avoid_cpus")
+	if raw == nil {
+		return nil
+	}
+	values, ok := raw.([]any)
+	if !ok {
+		return fmt.Errorf("tun.avoid_cpus must be a list of non-negative integers")
+	}
+	cpus := make([]int, 0, len(values))
+	for _, value := range values {
+		cpu, ok := value.(int)
+		if !ok || cpu < 0 {
+			return fmt.Errorf("tun.avoid_cpus must contain non-negative integers, got %v", value)
+		}
+		cpus = append(cpus, cpu)
+	}
+	if len(cpus) == 0 {
+		return nil
+	}
+	if err := util.AvoidCPUs(cpus, !configTest); err != nil {
+		if errors.Is(err, errors.ErrUnsupported) {
+			l.Warn("tun.avoid_cpus is only supported on Linux; ignoring")
+			return nil
+		}
+		return fmt.Errorf("tun.avoid_cpus: %w", err)
+	}
+	if !configTest {
+		l.Info("Excluded CPUs from process threads", "cpus", cpus)
+	}
+	return nil
+}
+
 func moduleVersion() string {
 	info, ok := debug.ReadBuildInfo()
 	if !ok {

main_test.go

@@ -59,6 +59,27 @@ func TestParseCpuAffinity(t *testing.T) {
 	}
 }
 
+func TestConfigureAvoidCPUs(t *testing.T) {
+	l := test.NewLogger()
+	for _, tc := range []struct {
+		value any
+		valid bool
+	}{
+		{nil, true}, {[]any{}, true}, {"auto", false}, {"1,2", false}, {[]any{-1}, false}, {[]any{1.5}, false},
+		{[]any{"2"}, false},
+	} {
+		c := config.NewC(l)
+		if tc.value != nil {
+			c.Settings["tun"] = map[string]any{"avoid_cpus": tc.value}
+		}
+		if err := configureAvoidCPUs(c, l, true); tc.valid {
+			require.NoError(t, err)
+		} else {
+			require.Error(t, err, "%v", tc.value)
+		}
+	}
+}
+
 // TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
 // when it hands back a Control to stop them with.
 //

util/cpuavoid_linux.go

@@ -0,0 +1,74 @@
+//go:build linux && !android && !e2e_testing
+
+package util
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// AvoidCPUs removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit
+// their parent's mask, so it repeats until a pass finds nothing left to change; a thread that exits meanwhile is
+// skipped. Every mask is checked before any is changed. With apply false it only validates, for configuration tests.
+// This affects the entire process, including other users of an embedded Nebula.
+func AvoidCPUs(cpus []int, apply bool) error {
+	const capacity = int(unsafe.Sizeof(unix.CPUSet{})) * 8
+	for _, cpu := range cpus {
+		if cpu < 0 || cpu >= capacity {
+			return fmt.Errorf("CPU %d is out of range", cpu)
+		}
+	}
+	if len(cpus) == 0 {
+		return nil
+	}
+	for {
+		masks, err := avoidingMasks(cpus)
+		if err != nil || !apply || len(masks) == 0 {
+			return err
+		}
+		for tid, mask := range masks {
+			if err := unix.SchedSetaffinity(tid, &mask); err != nil && !errors.Is(err, unix.ESRCH) {
+				return fmt.Errorf("set thread %d affinity: %w", tid, err)
+			}
+		}
+	}
+}
+
+// avoidingMasks returns the mask without cpus of every thread whose mask has any of them.
+func avoidingMasks(cpus []int) (map[int]unix.CPUSet, error) {
+	entries, err := os.ReadDir("/proc/self/task")
+	if err != nil {
+		return nil, err
+	}
+	masks := make(map[int]unix.CPUSet, len(entries))
+	for _, entry := range entries {
+		tid, err := strconv.Atoi(entry.Name())
+		if err != nil {
+			return nil, err
+		}
+		var mask unix.CPUSet
+		err = unix.SchedGetaffinity(tid, &mask)
+		if errors.Is(err, unix.ESRCH) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read thread %d affinity: %w", tid, err)
+		}
+		next := mask
+		for _, cpu := range cpus {
+			next.Clear(cpu)
+		}
+		if next.Count() == 0 {
+			return nil, fmt.Errorf("avoid_cpus leaves thread %d with no allowed CPU", tid)
+		}
+		if next != mask {
+			masks[tid] = next
+		}
+	}
+	return masks, nil
+}

util/cpuavoid_linux_test.go

@@ -0,0 +1,97 @@
+//go:build linux && !android && !e2e_testing
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"sync"
+	"testing"
+	"unsafe"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// TestAvoidCPUs changes the affinity of the whole process, so it runs in a child, leaving the test runner untouched.
+func TestAvoidCPUs(t *testing.T) {
+	if os.Getenv("NEBULA_AVOID_CPU_TEST") != "1" {
+		exe, err := os.Executable()
+		require.NoError(t, err)
+		cmd := exec.Command(exe, "-test.run=^TestAvoidCPUs$", "-test.v")
+		cmd.Env = append(os.Environ(), "NEBULA_AVOID_CPU_TEST=1")
+		out, err := cmd.CombinedOutput()
+		require.NoError(t, err, "%s", out)
+		return
+	}
+	allowed, err := AllowedCPUs()
+	require.NoError(t, err)
+	if len(allowed) < 2 {
+		t.Skip("requires at least two allowed CPUs")
+	}
+	cpu := allowed[0]
+	release := make(chan struct{})
+	var wg sync.WaitGroup
+	// lockThreads occupies 8 OS threads until release, so the runtime has to start more
+	lockThreads := func() {
+		var ready sync.WaitGroup
+		for range 8 {
+			ready.Add(1)
+			wg.Go(func() {
+				runtime.LockOSThread()
+				ready.Done()
+				<-release
+			})
+		}
+		ready.Wait()
+	}
+	lockThreads()
+
+	require.NoError(t, AvoidCPUs([]int{cpu}, false))
+	after, err := AllowedCPUs()
+	require.NoError(t, err)
+	require.Equal(t, allowed, after, "configuration check must not mutate affinity")
+	require.Error(t, AvoidCPUs(allowed, true), "must reject an empty mask before applying")
+	require.NoError(t, AvoidCPUs([]int{cpu, cpu}, true))
+	requireAvoided(t, cpu, allowed)
+	// Existing threads stay locked while new ones inherit the restriction
+	lockThreads()
+	requireAvoided(t, cpu, allowed)
+	close(release)
+	wg.Wait()
+}
+
+// requireAvoided checks no thread may run on cpu, and every thread still may on some CPU of allowed only.
+func requireAvoided(t *testing.T, cpu int, allowed []int) {
+	t.Helper()
+	entries, err := os.ReadDir("/proc/self/task")
+	require.NoError(t, err)
+	for _, entry := range entries {
+		tid, err := strconv.Atoi(entry.Name())
+		require.NoError(t, err)
+		var mask unix.CPUSet
+		err = unix.SchedGetaffinity(tid, &mask)
+		if err == unix.ESRCH {
+			continue
+		}
+		require.NoError(t, err)
+		require.False(t, mask.IsSet(cpu), "thread %d", tid)
+		require.Positive(t, mask.Count())
+		for id := range len(mask) * 64 {
+			if mask.IsSet(id) {
+				require.Contains(t, allowed, id, "must not widen the inherited mask")
+			}
+		}
+	}
+}
+
+// TestAvoidCPUsRange checks the bounds of unix.CPUSet, whose words are 32 bits wide on 32-bit platforms.
+func TestAvoidCPUsRange(t *testing.T) {
+	capacity := int(unsafe.Sizeof(unix.CPUSet{})) * 8
+	require.NoError(t, AvoidCPUs([]int{capacity - 1}, false))
+	require.EqualError(t, AvoidCPUs([]int{capacity}, false), fmt.Sprintf("CPU %d is out of range", capacity))
+	require.EqualError(t, AvoidCPUs([]int{-1}, false), "CPU -1 is out of range")
+}

util/cpuavoid_other.go

@@ -0,0 +1,8 @@
+//go:build !linux || android || e2e_testing
+
+package util
+
+import "errors"
+
+// AvoidCPUs is unsupported outside Linux: there is no thread affinity API to restrict.
+func AvoidCPUs(_ []int, _ bool) error { return errors.ErrUnsupported }