tun: default pin CPUs avoid NIC IRQ cores

When tun.pin_threads is on and tun.cpu_affinity is unset, pick pin CPUs
from the allowed set that do not service any up physical NIC's MSI vectors
(read-only walk of /sys/class/net/*/device/msi_irqs and
/proc/irq/*/effective_affinity_list). The old allowed[i] default pinned
encrypt threads onto exactly the cores drivers affine their first RX queue
IRQs to; a flow whose RSS queue fired there had NAPI fighting encrypt for
the core (measured 8.4 vs 10.2 Gbps REV bimodality). Falls back to the old
spread, with a log, when there aren't enough IRQ-free CPUs - e.g. drivers
that allocate one queue per core until the admin narrows them (ethtool -L).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

examples/config.yml

@@ -257,13 +257,21 @@ tun:
   # Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's
   # batched sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire
   # instead of being sprayed across multiple TX rings and reordered. Not reloadable.
+  #
+  # When cpu_affinity is unset, nebula picks CPUs that do NOT service any physical NIC's interrupts (read from
+  # /sys/class/net/*/device/msi_irqs and /proc/irq/*/effective_affinity_list): an encrypt thread pinned onto a core
+  # that also runs NAPI for a NIC RX queue fights the softirq for the core and collapses throughput for flows hashed
+  # to that queue. If the NIC's vectors blanket every allowed CPU (many drivers default to one queue per core) the
+  # avoidance logs and falls back to the old spread; narrow the NIC's queue/IRQ spread (e.g. `ethtool -X <dev>
+  # equal N`) or set cpu_affinity explicitly to benefit.
   #pin_threads: true
 
   # Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine
   # (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra
   # entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
   # a non-integer or not-allowed entry disables the override and falls back to spreading queues across the allowed
-  # CPUs. Only meaningful while pin_threads is true. Not reloadable.
+  # CPUs. Setting this disables the automatic NIC-IRQ avoidance described under pin_threads — prefer CPUs that don't
+  # service your underlay NIC's RX queue IRQs. Only meaningful while pin_threads is true. Not reloadable.
   #cpu_affinity:
   #  - 2
   #  - 4

main.go

@@ -214,6 +214,12 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		l.Warn("Failed to start DNS responder", "error", err)
 	}
 
+	pinThreads := c.GetBool("tun.pin_threads", true)
+	cpuAffinity := parseCpuAffinity(c, l, routines)
+	if pinThreads && len(cpuAffinity) == 0 && !configTest {
+		cpuAffinity = defaultCPUAffinityAvoidingIRQs(l, routines)
+	}
+
 	ifConfig := &InterfaceConfig{
 		HostMap:               hostMap,
 		Inside:                tun,
@@ -235,8 +241,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),
 		punchy:                punchy,
 		ConntrackCacheTimeout: conntrackCacheTimeout,
-		CpuAffinity:           parseCpuAffinity(c, l, routines),
-		PinThreads:            c.GetBool("tun.pin_threads", true),
+		CpuAffinity:           cpuAffinity,
+		PinThreads:            pinThreads,
 		l:                     l,
 	}
 
@@ -353,6 +359,57 @@ func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
 	return cpus
 }
 
+// defaultCPUAffinityAvoidingIRQs picks the default pin set for the tun
+// readers when tun.cpu_affinity is unset: allowed CPUs that do NOT service
+// any physical NIC's interrupts. The stock allowed[i] spread pins the
+// encrypt threads onto exactly the cores most drivers affine their first RX
+// queue IRQs to, so whenever a flow's RSS queue fires on a core hosting a
+// tun reader, NAPI and encrypt fight for the core and per-flow throughput
+// drops (measured: REV 8.4 vs 10.2 Gbps on the same hardware, 2026-07-14).
+//
+// Returns nil — keeping the old allowed[i] fallback in listenIn — when IRQ
+// info is unavailable or when there aren't enough IRQ-free CPUs to give
+// every routine its own core: silently doubling readers up on fewer cores
+// is worse than the occasional IRQ collision. NICs whose vectors blanket
+// every CPU (e.g. mlx5 defaults to one queue per core) make avoidance
+// impossible; narrowing the NIC's spread (ethtool -X <dev> equal N, or
+// /proc/irq/*/smp_affinity) or setting tun.cpu_affinity explicitly makes it
+// effective.
+func defaultCPUAffinityAvoidingIRQs(l *slog.Logger, routines int) []int {
+	irq, err := util.NICIRQCPUs()
+	if err != nil || len(irq) == 0 {
+		return nil
+	}
+	allowed, err := util.AllowedCPUs()
+	if err != nil {
+		return nil
+	}
+	cpus := chooseIRQFreeCPUs(allowed, irq, routines)
+	if cpus == nil {
+		l.Info("not enough CPUs are free of NIC IRQs to give every tun reader its own; using the default spread",
+			"routines", routines, "allowed", len(allowed), "irqCPUs", len(irq))
+		return nil
+	}
+	l.Info("pinning tun readers to CPUs clear of NIC IRQs", "cpus", cpus)
+	return cpus
+}
+
+// chooseIRQFreeCPUs returns the first `routines` allowed CPUs not present in
+// irq, or nil if fewer than `routines` qualify.
+func chooseIRQFreeCPUs(allowed []int, irq map[int]bool, routines int) []int {
+	free := make([]int, 0, routines)
+	for _, cpu := range allowed {
+		if irq[cpu] {
+			continue
+		}
+		free = append(free, cpu)
+		if len(free) == routines {
+			return free
+		}
+	}
+	return nil
+}
+
 func moduleVersion() string {
 	info, ok := debug.ReadBuildInfo()
 	if !ok {

main_test.go

@@ -9,6 +9,26 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestChooseIRQFreeCPUs(t *testing.T) {
+	irq := map[int]bool{0: true, 1: true, 2: true, 3: true}
+
+	// Plenty of IRQ-free CPUs: take the first `routines` of them in order.
+	assert.Equal(t, []int{4, 5}, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4, 5, 6}, irq, 2))
+
+	// Exactly enough.
+	assert.Equal(t, []int{4, 5, 6}, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4, 5, 6}, irq, 3))
+
+	// Not enough IRQ-free CPUs: nil, caller keeps the old default rather
+	// than doubling readers up on shared cores.
+	assert.Nil(t, chooseIRQFreeCPUs([]int{0, 1, 2, 3, 4}, irq, 2))
+
+	// No IRQ info at all behaves like a plain prefix of allowed.
+	assert.Equal(t, []int{0, 1}, chooseIRQFreeCPUs([]int{0, 1, 2}, map[int]bool{}, 2))
+
+	// Non-contiguous allowed set (cgroup cpuset) with holes.
+	assert.Equal(t, []int{9, 12}, chooseIRQFreeCPUs([]int{1, 3, 9, 12}, map[int]bool{1: true, 3: true}, 2))
+}
+
 func TestParseCpuAffinity(t *testing.T) {
 	l := test.NewLogger()
 

util/irqcpus_linux.go

@@ -0,0 +1,198 @@
+//go:build linux && !android && !e2e_testing
+
+package util
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// NICIRQCPUs returns the set of CPUs that service interrupts for the ACTIVE
+// RX/TX queues of physical network interfaces that are up. Read-only: it
+// matches /proc/interrupts action names against each NIC's PCI address and
+// interface name, drops vectors whose queue index is beyond the device's
+// active queue count (drivers like mlx5 keep handlers registered for
+// deactivated queues, so /proc/interrupts alone over-reports), and unions
+// /proc/irq/<n>/effective_affinity_list for the survivors.
+//
+// Callers use this to keep busy pinned threads OFF those CPUs: a thread
+// pinned onto a core that also runs NAPI for a NIC RX queue competes with
+// softirq processing for the core and measurably collapses throughput for
+// flows hashed to that queue.
+func NICIRQCPUs() (map[int]bool, error) {
+	return nicIRQCPUs("/sys/class/net", "/proc/irq", "/proc/interrupts")
+}
+
+// irqAction is one row of /proc/interrupts: the IRQ number and the action
+// (handler) name in its final column, e.g. "mlx5_comp3@pci:0000:82:00.0".
+type irqAction struct {
+	irq    string
+	action string
+}
+
+func nicIRQCPUs(netDir, irqDir, interruptsPath string) (map[int]bool, error) {
+	actions, err := parseInterrupts(interruptsPath)
+	if err != nil {
+		return nil, err
+	}
+	devs, err := os.ReadDir(netDir)
+	if err != nil {
+		return nil, err
+	}
+
+	cpus := make(map[int]bool)
+	for _, dev := range devs {
+		devPath := filepath.Join(netDir, dev.Name())
+		pciDev, err := filepath.EvalSymlinks(filepath.Join(devPath, "device"))
+		if err != nil {
+			continue // virtual device (lo, tun, bridge, vlan, ...)
+		}
+		pciAddr := filepath.Base(pciDev)
+		state, err := os.ReadFile(filepath.Join(devPath, "operstate"))
+		if err != nil || strings.TrimSpace(string(state)) != "up" {
+			continue // a down NIC's queue IRQs don't fire
+		}
+		nq := countQueues(filepath.Join(devPath, "queues"))
+
+		for _, ia := range actions {
+			if !strings.Contains(ia.action, pciAddr) && !containsWord(ia.action, dev.Name()) {
+				continue
+			}
+			// Vector naming puts the queue index at the end of the handler
+			// name (mlx5_comp3@pci:..., ice-eth0-TxRx-3, virtio0-input.3).
+			// An index at or beyond the active queue count is a handler for
+			// a deactivated queue: registered, but it will not fire.
+			name, _, _ := strings.Cut(ia.action, "@")
+			if idx, ok := trailingInt(name); ok && idx >= nq {
+				continue
+			}
+			for _, cpu := range irqAffinity(irqDir, ia.irq) {
+				cpus[cpu] = true
+			}
+		}
+	}
+	return cpus, nil
+}
+
+// parseInterrupts extracts (irq, action) pairs from /proc/interrupts,
+// skipping the header and the non-numeric summary rows (NMI, LOC, ...).
+func parseInterrupts(path string) ([]irqAction, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []irqAction
+	for line := range strings.SplitSeq(string(data), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		irq, ok := strings.CutSuffix(fields[0], ":")
+		if !ok {
+			continue
+		}
+		if _, err := strconv.Atoi(irq); err != nil {
+			continue
+		}
+		out = append(out, irqAction{irq: irq, action: fields[len(fields)-1]})
+	}
+	return out, nil
+}
+
+// irqAffinity returns the CPUs IRQ n actually targets.
+// effective_affinity_list is the vector's real target; smp_affinity_list
+// (the fallback for kernels without effective affinity reporting) is the
+// admin-allowed mask and may be wider.
+func irqAffinity(irqDir, irq string) []int {
+	irqPath := filepath.Join(irqDir, irq)
+	list, err := os.ReadFile(filepath.Join(irqPath, "effective_affinity_list"))
+	if err != nil || len(strings.TrimSpace(string(list))) == 0 {
+		list, err = os.ReadFile(filepath.Join(irqPath, "smp_affinity_list"))
+		if err != nil {
+			return nil
+		}
+	}
+	return parseCPUList(strings.TrimSpace(string(list)))
+}
+
+// countQueues counts the rx-* entries of a netdev's queues directory — the
+// device's ACTIVE RX queues (sysfs removes the directories when a queue is
+// deactivated, e.g. by ethtool -L).
+func countQueues(queuesDir string) int {
+	entries, err := os.ReadDir(queuesDir)
+	if err != nil {
+		return 0
+	}
+	n := 0
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), "rx-") {
+			n++
+		}
+	}
+	return n
+}
+
+// containsWord reports whether s contains word bounded by non-alphanumeric
+// characters (or string edges), so ifname "eth0" doesn't match "eth01".
+func containsWord(s, word string) bool {
+	for start := 0; ; {
+		i := strings.Index(s[start:], word)
+		if i < 0 {
+			return false
+		}
+		i += start
+		before := i == 0 || !isAlnum(s[i-1])
+		afterIdx := i + len(word)
+		after := afterIdx == len(s) || !isAlnum(s[afterIdx])
+		if before && after {
+			return true
+		}
+		start = i + 1
+	}
+}
+
+func isAlnum(b byte) bool {
+	return b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z'
+}
+
+// trailingInt parses the decimal digits at the end of s.
+func trailingInt(s string) (int, bool) {
+	i := len(s)
+	for i > 0 && s[i-1] >= '0' && s[i-1] <= '9' {
+		i--
+	}
+	if i == len(s) {
+		return 0, false
+	}
+	n, err := strconv.Atoi(s[i:])
+	return n, err == nil
+}
+
+// parseCPUList parses the kernel's cpulist format: comma-separated CPU ids
+// or inclusive ranges, e.g. "0-3,8,10-12". Malformed elements are skipped —
+// this parses trusted kernel output, not user input.
+func parseCPUList(s string) []int {
+	if s == "" {
+		return nil
+	}
+	var cpus []int
+	for part := range strings.SplitSeq(s, ",") {
+		lo, hi, ok := strings.Cut(part, "-")
+		start, err := strconv.Atoi(strings.TrimSpace(lo))
+		if err != nil {
+			continue
+		}
+		end := start
+		if ok {
+			if end, err = strconv.Atoi(strings.TrimSpace(hi)); err != nil {
+				continue
+			}
+		}
+		for cpu := start; cpu <= end; cpu++ {
+			cpus = append(cpus, cpu)
+		}
+	}
+	return cpus
+}

util/irqcpus_linux_test.go

@@ -0,0 +1,153 @@
+//go:build linux && !android && !e2e_testing
+
+package util
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"testing"
+)
+
+func TestParseCPUList(t *testing.T) {
+	cases := []struct {
+		in   string
+		want []int
+	}{
+		{"", nil},
+		{"3", []int{3}},
+		{"0-3", []int{0, 1, 2, 3}},
+		{"0-2,8,10-11", []int{0, 1, 2, 8, 10, 11}},
+		{"garbage", nil},
+		{"1,garbage,4", []int{1, 4}},
+	}
+	for _, c := range cases {
+		if got := parseCPUList(c.in); !reflect.DeepEqual(got, c.want) {
+			t.Errorf("parseCPUList(%q) = %v, want %v", c.in, got, c.want)
+		}
+	}
+}
+
+func TestTrailingInt(t *testing.T) {
+	cases := []struct {
+		in   string
+		want int
+		ok   bool
+	}{
+		{"mlx5_comp12", 12, true},
+		{"ice-eth0-TxRx-3", 3, true},
+		{"virtio0-input.7", 7, true},
+		{"mlx5_async0", 0, true},
+		{"no-digits", 0, false},
+		{"", 0, false},
+	}
+	for _, c := range cases {
+		got, ok := trailingInt(c.in)
+		if got != c.want || ok != c.ok {
+			t.Errorf("trailingInt(%q) = (%d, %v), want (%d, %v)", c.in, got, ok, c.want, c.ok)
+		}
+	}
+}
+
+func TestContainsWord(t *testing.T) {
+	if !containsWord("ice-eth0-TxRx-3", "eth0") {
+		t.Error("eth0 should match with boundaries")
+	}
+	if containsWord("ice-eth01-TxRx-3", "eth0") {
+		t.Error("eth0 must not match inside eth01")
+	}
+	if !containsWord("eth0", "eth0") {
+		t.Error("exact match should work")
+	}
+}
+
+// fakeNIC builds /sys/class/net/<name> with operstate, a device symlink to a
+// PCI-address-named dir (physical NICs only), and nq rx queue directories.
+func fakeNIC(t *testing.T, netDir, name, operstate, pciAddr string, nq int) {
+	t.Helper()
+	devPath := filepath.Join(netDir, name)
+	if err := os.MkdirAll(devPath, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(devPath, "operstate"), []byte(operstate+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if pciAddr == "" {
+		return
+	}
+	pciDir := filepath.Join(netDir, "..", "devices", pciAddr)
+	if err := os.MkdirAll(pciDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink(pciDir, filepath.Join(devPath, "device")); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < nq; i++ {
+		if err := os.MkdirAll(filepath.Join(devPath, "queues", "rx-"+strconv.Itoa(i)), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func writeIRQ(t *testing.T, irqDir, irq, affinity string) {
+	t.Helper()
+	p := filepath.Join(irqDir, irq)
+	if err := os.MkdirAll(p, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(p, "effective_affinity_list"), []byte(affinity+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestNICIRQCPUs(t *testing.T) {
+	root := t.TempDir()
+	netDir := filepath.Join(root, "class", "net")
+	irqDir := filepath.Join(root, "irq")
+	if err := os.MkdirAll(netDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	// eth0: up, 2 active queues at 0000:82:00.0. comp0/comp1 active,
+	// comp2 is a deactivated queue's leftover handler, async0 always fires.
+	fakeNIC(t, netDir, "eth0", "up", "0000:82:00.0", 2)
+	// eth1: physical but down; its vectors must not count.
+	fakeNIC(t, netDir, "eth1", "down", "0000:83:00.0", 2)
+	// eth9: up, matched by ifname (intel-style action names), 1 queue.
+	fakeNIC(t, netDir, "eth9", "up", "0000:84:00.0", 1)
+	// nebula1: virtual, no device dir.
+	fakeNIC(t, netDir, "nebula1", "up", "", 0)
+
+	interrupts := filepath.Join(root, "interrupts")
+	content := `           CPU0       CPU1
+ 100:   1 2   IR-PCI-MSIX  1-edge  mlx5_comp0@pci:0000:82:00.0
+ 101:   1 2   IR-PCI-MSIX  2-edge  mlx5_comp1@pci:0000:82:00.0
+ 102:   1 2   IR-PCI-MSIX  3-edge  mlx5_comp2@pci:0000:82:00.0
+ 103:   1 2   IR-PCI-MSIX  4-edge  mlx5_async0@pci:0000:82:00.0
+ 200:   1 2   IR-PCI-MSIX  5-edge  mlx5_comp0@pci:0000:83:00.0
+ 300:   1 2   IR-PCI-MSIX  6-edge  ice-eth9-TxRx-0
+ 301:   1 2   IR-PCI-MSIX  7-edge  ice-eth9-TxRx-1
+ NMI:   0 0   Non-maskable interrupts
+`
+	if err := os.WriteFile(interrupts, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	writeIRQ(t, irqDir, "100", "0-1") // eth0 comp0: counted
+	writeIRQ(t, irqDir, "101", "2")   // eth0 comp1: counted
+	writeIRQ(t, irqDir, "102", "5")   // eth0 comp2: beyond 2 queues, skipped
+	writeIRQ(t, irqDir, "103", "7")   // eth0 async0: counted
+	writeIRQ(t, irqDir, "200", "9")   // eth1 down: skipped
+	writeIRQ(t, irqDir, "300", "11")  // eth9 TxRx-0: counted
+	writeIRQ(t, irqDir, "301", "12")  // eth9 TxRx-1: beyond 1 queue, skipped
+
+	got, err := nicIRQCPUs(netDir, irqDir, interrupts)
+	if err != nil {
+		t.Fatalf("nicIRQCPUs: %v", err)
+	}
+	want := map[int]bool{0: true, 1: true, 2: true, 7: true, 11: true}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("nicIRQCPUs = %v, want %v", got, want)
+	}
+}

util/irqcpus_other.go

@@ -0,0 +1,9 @@
+//go:build !linux || android || e2e_testing
+
+package util
+
+// NICIRQCPUs reports no IRQ information on platforms without the linux
+// sysfs interface; callers fall back to their non-IRQ-aware defaults.
+func NICIRQCPUs() (map[int]bool, error) {
+	return nil, nil
+}