wakey wakey

connection_manager.go

@@ -44,6 +44,11 @@ type connectionManager struct {
 	inactivityTimeout       atomic.Int64
 	dropInactive            atomic.Bool
 
+	// Wake-from-sleep handling, sampled once per tick in Start
+	wakeDetector       *wakeDetector
+	clearOnWake        atomic.Bool
+	wakeClearThreshold atomic.Int64
+
 	l *slog.Logger
 }
 
@@ -54,6 +59,7 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p
 		punchy:        p,
 		relayUsed:     make(map[uint32]struct{}),
 		relayUsedLock: &sync.RWMutex{},
+		wakeDetector:  newWakeDetector(),
 	}
 
 	cm.reload(c, true)
@@ -98,12 +104,38 @@ func (cm *connectionManager) reload(c *config.C, initial bool) {
 			)
 		}
 	}
+
+	if initial || c.HasChanged("tunnels.clear_on_wake") {
+		old := cm.clearOnWake.Load()
+		cm.clearOnWake.Store(c.GetBool("tunnels.clear_on_wake", true))
+		if !initial {
+			cm.l.Info("Clear on wake setting has changed",
+				"oldBool", old,
+				"newBool", cm.clearOnWake.Load(),
+			)
+		}
+	}
+
+	if initial || c.HasChanged("tunnels.wake_clear_threshold") {
+		old := cm.getWakeClearThreshold()
+		cm.wakeClearThreshold.Store((int64)(c.GetDuration("tunnels.wake_clear_threshold", 30*time.Second)))
+		if !initial {
+			cm.l.Info("Wake clear threshold has changed",
+				"oldDuration", old,
+				"newDuration", cm.getWakeClearThreshold(),
+			)
+		}
+	}
 }
 
 func (cm *connectionManager) getInactivityTimeout() time.Duration {
 	return (time.Duration)(cm.inactivityTimeout.Load())
 }
 
+func (cm *connectionManager) getWakeClearThreshold() time.Duration {
+	return (time.Duration)(cm.wakeClearThreshold.Load())
+}
+
 func (cm *connectionManager) In(h *HostInfo) {
 	h.in.Store(true)
 }
@@ -136,6 +168,73 @@ func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time)
 	return in, out
 }
 
+// checkWake runs once per tick and clears every tunnel when the machine has just returned from system sleep.
+// Tunnels rarely survive a suspend: our NAT mappings have expired and our address has usually changed, so every
+// established hostinfo is a corpse that will eat 15-20s of traffic checks before the wheel declares it dead.
+// Clearing now means the first packet after wake starts a fresh handshake immediately.
+//
+// The suspend itself costs nothing here: the ticker driving us is frozen with the rest of the process and this
+// fires within one tick of resume.
+func (cm *connectionManager) checkWake() {
+	slept, ok := cm.wakeDetector.Sample()
+	if !ok || slept == 0 {
+		return
+	}
+
+	// The clock pair is read non-atomically, so scheduling jitter shows up as tiny sub-millisecond "sleeps".
+	// Keep the floor well above that so a zero/nonsense threshold can't clear tunnels on every tick.
+	threshold := max(cm.getWakeClearThreshold(), time.Second)
+
+	if slept < threshold {
+		// Short suspends (lid closed and quickly reopened) often come back before NAT state expires; those
+		// tunnels may well be alive, leave them to the normal traffic checks.
+		if slept >= time.Second {
+			cm.l.Debug("Woke from sleep below the clear threshold, leaving tunnels alone",
+				"sleptFor", slept,
+				"threshold", threshold,
+			)
+		}
+		return
+	}
+
+	if !cm.clearOnWake.Load() {
+		cm.l.Info("Woke from sleep, tunnels.clear_on_wake is disabled so tunnels are left to the normal traffic checks", "sleptFor", slept)
+		return
+	}
+
+	closed := cm.clearAllTunnels()
+	cm.l.Info("Woke from sleep, cleared tunnels", "sleptFor", slept, "tunnelsCleared", closed)
+
+	// Our public address almost certainly changed; get it to the lighthouses as soon as possible so peers can
+	// find us again. The update rides over a fresh lighthouse handshake. If the network isn't back up yet these
+	// sends fail harmlessly and the periodic update worker retries within lighthouse.interval.
+	cm.intf.lightHouse.TriggerUpdate()
+}
+
+// clearAllTunnels closes every tunnel in the hostmap locally, without notifying the remotes. It is the wake-from-
+// sleep counterpart to Control.CloseAllTunnels: after a suspend the remotes stopped hearing from us long ago, and
+// close packets fired into a network that may not even be up yet are wasted, so we only tear down our own state
+// and let the next packet to each host start a fresh handshake.
+func (cm *connectionManager) clearAllTunnels() int {
+	cm.hostMap.RLock()
+	hostinfos := make([]*HostInfo, 0, len(cm.hostMap.Indexes))
+	for _, h := range cm.hostMap.Indexes {
+		hostinfos = append(hostinfos, h)
+	}
+	cm.hostMap.RUnlock()
+
+	for _, h := range hostinfos {
+		cm.intf.closeTunnel(h)
+	}
+
+	// With every tunnel gone no relay can be in use, drop the usage tracking wholesale.
+	cm.relayUsedLock.Lock()
+	clear(cm.relayUsed)
+	cm.relayUsedLock.Unlock()
+
+	return len(hostinfos)
+}
+
 func (cm *connectionManager) Start(ctx context.Context) {
 	clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration)
 	defer clockSource.Stop()
@@ -150,6 +249,7 @@ func (cm *connectionManager) Start(ctx context.Context) {
 			return
 
 		case now := <-clockSource.C:
+			cm.checkWake()
 			cm.trafficTimer.Advance(now)
 			for {
 				localIndex, has := cm.trafficTimer.Purge()

connection_manager_test.go

@@ -501,3 +501,85 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
 func (d *dummyCert) Copy() cert.Certificate {
 	return d
 }
+
+func TestConnectionManager_WakeClear(t *testing.T) {
+	l := test.NewLogger()
+	localrange := netip.MustParsePrefix("10.1.1.1/24")
+	vpnIp := netip.MustParseAddr("172.1.1.2")
+	preferredRanges := []netip.Prefix{localrange}
+
+	// Very incomplete mock objects
+	hostMap := newHostMap(l)
+	hostMap.preferredRanges.Store(&preferredRanges)
+
+	cs := &CertState{
+		initiatingVersion: cert.Version1,
+		privateKey:        []byte{},
+		v1Cert:            &dummyCert{version: cert.Version1},
+		v1Credential:      nil,
+	}
+
+	lh := newTestLighthouse()
+	ifce := &Interface{
+		hostMap:          hostMap,
+		inside:           &overlaytest.NoopTun{},
+		outside:          &udp.NoopConn{},
+		firewall:         &Firewall{},
+		lightHouse:       lh,
+		pki:              &PKI{},
+		handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
+		l:                l,
+	}
+	ifce.pki.cs.Store(cs)
+
+	// Create manager
+	conf := config.NewC(test.NewLogger())
+	punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
+	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
+	nc.intf = ifce
+
+	// Drive the wake detector from a fake clock pair
+	suspended := time.Duration(0)
+	nc.wakeDetector = &wakeDetector{read: func() (time.Duration, bool) { return suspended, true }}
+	nc.checkWake() // primes the baseline
+
+	addTunnel := func(localIndex uint32) *HostInfo {
+		hostinfo := &HostInfo{
+			vpnAddrs:      []netip.Addr{vpnIp},
+			localIndexId:  localIndex,
+			remoteIndexId: 9901,
+		}
+		hostinfo.ConnectionState = &ConnectionState{
+			myCert: &dummyCert{version: cert.Version1},
+		}
+		nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
+		return hostinfo
+	}
+
+	addTunnel(1099)
+	nc.RelayUsed(5000)
+
+	// No suspend, nothing happens
+	nc.checkWake()
+	assert.Contains(t, nc.hostMap.Indexes, uint32(1099))
+
+	// A suspend below the threshold leaves tunnels alone
+	suspended += 5 * time.Second
+	nc.checkWake()
+	assert.Contains(t, nc.hostMap.Indexes, uint32(1099))
+
+	// A suspend past the threshold clears everything, including relay usage tracking
+	suspended += time.Hour
+	nc.checkWake()
+	assert.Empty(t, nc.hostMap.Indexes)
+	assert.Empty(t, nc.hostMap.Hosts)
+	assert.Empty(t, nc.relayUsed)
+
+	// With clear_on_wake disabled the tunnels survive a long suspend
+	addTunnel(1100)
+	nc.clearOnWake.Store(false)
+	suspended += time.Hour
+	nc.checkWake()
+	assert.Contains(t, nc.hostMap.Indexes, uint32(1100))
+	assert.Contains(t, nc.hostMap.Hosts, vpnIp)
+}

examples/config.yml

@@ -390,6 +390,20 @@ logging:
   # This setting is reloadable
   #inactivity_timeout: 10m
 
+  # clear_on_wake controls whether all tunnels are immediately torn down (locally, without notifying the remotes)
+  # when the machine detects it has just woken from system sleep. Tunnels rarely survive a suspend: NAT mappings
+  # expire and the machine's address usually changes, so waiting for the normal liveness checks costs 15-20 seconds
+  # of black-holed traffic per tunnel after wake. Clearing them means the first packet after wake starts a fresh
+  # handshake right away.
+  # This setting is reloadable
+  #clear_on_wake: true
+
+  # wake_clear_threshold is the minimum time the machine must have been suspended for clear_on_wake to act.
+  # Suspends shorter than this often come back before NAT state expires, so those tunnels may still be alive and
+  # are left to the normal liveness checks. Values below 1s are treated as 1s.
+  # This setting is reloadable
+  #wake_clear_threshold: 30s
+
 # Nebula security group configuration
 firewall:
   # Action to take when a packet is not allowed by the firewall rules.

wake_clock_darwin.go

@@ -0,0 +1,29 @@
+//go:build darwin
+
+package nebula
+
+import (
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// suspendClockDelta returns CLOCK_MONOTONIC - CLOCK_UPTIME_RAW. On macOS CLOCK_MONOTONIC keeps counting across
+// system sleep while CLOCK_UPTIME_RAW (mach_absolute_time) pauses, so the difference grows by time spent asleep.
+//
+// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the
+// wakeDetector clamps out the noise.
+//
+// Caveat: on Apple Silicon the hardware timebase keeps ticking through sleep, which can make both clocks advance
+// and the spread stay flat, leaving this detector blind. That fails safe (no clears, behavior as before); IOKit
+// power notifications are the follow-up for full coverage on those machines.
+func suspendClockDelta() (time.Duration, bool) {
+	var uptime, mono unix.Timespec
+	if err := unix.ClockGettime(unix.CLOCK_UPTIME_RAW, &uptime); err != nil {
+		return 0, false
+	}
+	if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil {
+		return 0, false
+	}
+	return time.Duration(mono.Nano() - uptime.Nano()), true
+}

wake_clock_generic.go

@@ -0,0 +1,11 @@
+//go:build !linux && !darwin && !windows
+
+package nebula
+
+import "time"
+
+// suspendClockDelta reports that this platform has no usable clock pair for detecting system sleep; the wake
+// detector stays dormant and dead tunnels are left to the normal traffic checks.
+func suspendClockDelta() (time.Duration, bool) {
+	return 0, false
+}

wake_clock_linux.go

@@ -0,0 +1,26 @@
+//go:build linux
+
+package nebula
+
+import (
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// suspendClockDelta returns CLOCK_BOOTTIME - CLOCK_MONOTONIC. CLOCK_MONOTONIC pauses while the system is suspended
+// and CLOCK_BOOTTIME does not, so the difference only ever grows, and only by time spent suspended. Both reads are
+// vDSO calls, cheap enough for a hot ticker.
+//
+// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the
+// wakeDetector clamps out the noise.
+func suspendClockDelta() (time.Duration, bool) {
+	var mono, boot unix.Timespec
+	if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil {
+		return 0, false
+	}
+	if err := unix.ClockGettime(unix.CLOCK_BOOTTIME, &boot); err != nil {
+		return 0, false
+	}
+	return time.Duration(boot.Nano() - mono.Nano()), true
+}

wake_clock_windows.go

@@ -0,0 +1,41 @@
+//go:build windows
+
+package nebula
+
+import (
+	"sync"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+var (
+	procQueryInterruptTime         = windows.NewLazySystemDLL("kernelbase.dll").NewProc("QueryInterruptTime")
+	procQueryUnbiasedInterruptTime = windows.NewLazySystemDLL("kernel32.dll").NewProc("QueryUnbiasedInterruptTime")
+
+	// QueryInterruptTime needs Windows 10; probe once and stay dormant on anything older.
+	wakeClockAvailable = sync.OnceValue(func() bool {
+		return procQueryInterruptTime.Find() == nil && procQueryUnbiasedInterruptTime.Find() == nil
+	})
+)
+
+// suspendClockDelta returns interrupt time minus unbiased interrupt time, both in 100ns units. The unbiased count
+// excludes time the system spends suspended while the biased one includes it, so the difference grows by exactly
+// the time spent asleep.
+//
+// The pausing (unbiased) clock is read first so scheduling jitter between the two reads biases the delta positive;
+// the wakeDetector clamps out the noise.
+func suspendClockDelta() (time.Duration, bool) {
+	if !wakeClockAvailable() {
+		return 0, false
+	}
+
+	var unbiased, biased uint64
+	if r1, _, _ := procQueryUnbiasedInterruptTime.Call(uintptr(unsafe.Pointer(&unbiased))); r1 == 0 {
+		return 0, false
+	}
+	// Returns void, cannot fail once resolved.
+	_, _, _ = procQueryInterruptTime.Call(uintptr(unsafe.Pointer(&biased)))
+	return time.Duration(int64(biased-unbiased)) * 100, true
+}

wake_detector.go

@@ -0,0 +1,47 @@
+package nebula
+
+import "time"
+
+// wakeDetector notices when the machine has returned from system sleep and measures how long it was suspended.
+//
+// It samples the spread between two kernel clocks: one that pauses across a suspend and one that keeps counting
+// (suspendClockDelta, per platform). While the machine is awake the spread is constant no matter how starved,
+// stopped, or stepped this process is — SIGSTOP, debugger pauses, scheduler starvation, and NTP adjustments move
+// both clocks together or neither, so none of them can fake a wake. A true suspend is the only thing that grows
+// the spread, and it grows by exactly the time spent suspended.
+//
+// Sample is intended to piggyback on a ticker the caller already runs; it costs two clock reads. It is not safe
+// for concurrent use.
+type wakeDetector struct {
+	// read returns the current spread between the two clocks, false if this platform can't provide one.
+	read   func() (time.Duration, bool)
+	last   time.Duration
+	primed bool
+}
+
+func newWakeDetector() *wakeDetector {
+	return &wakeDetector{read: suspendClockDelta}
+}
+
+// Sample returns how long the machine was suspended since the previous call, 0 if it wasn't, and false if the
+// platform has no way to tell. The first call primes the baseline and always reports 0.
+func (w *wakeDetector) Sample() (time.Duration, bool) {
+	delta, ok := w.read()
+	if !ok {
+		return 0, false
+	}
+
+	if !w.primed {
+		w.primed = true
+		w.last = delta
+		return 0, true
+	}
+
+	slept := delta - w.last
+	w.last = delta
+	if slept < 0 {
+		// The clock pair is read non-atomically so tiny negative jitter is possible; it is never a wake.
+		slept = 0
+	}
+	return slept, true
+}

wake_detector_test.go

@@ -0,0 +1,67 @@
+package nebula
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWakeDetector(t *testing.T) {
+	delta := time.Duration(0)
+	ok := true
+	w := &wakeDetector{read: func() (time.Duration, bool) { return delta, ok }}
+
+	// The first sample primes the baseline and never reports a wake, even with a pre-existing spread
+	delta = 3 * time.Hour
+	slept, sok := w.Sample()
+	assert.True(t, sok)
+	assert.Equal(t, time.Duration(0), slept)
+
+	// A stable spread means the machine never slept
+	slept, sok = w.Sample()
+	assert.True(t, sok)
+	assert.Equal(t, time.Duration(0), slept)
+
+	// The spread grows by exactly the time spent suspended
+	delta += 42 * time.Second
+	slept, sok = w.Sample()
+	assert.True(t, sok)
+	assert.Equal(t, 42*time.Second, slept)
+
+	// A wake is reported once, then the baseline moves with it
+	slept, sok = w.Sample()
+	assert.True(t, sok)
+	assert.Equal(t, time.Duration(0), slept)
+
+	// Negative jitter from the non-atomic clock pair reads clamps to zero
+	delta -= time.Microsecond
+	slept, sok = w.Sample()
+	assert.True(t, sok)
+	assert.Equal(t, time.Duration(0), slept)
+
+	// Consecutive suspends both report; the clamped jitter moved the baseline so it is not double-counted
+	delta += time.Minute
+	slept, _ = w.Sample()
+	assert.Equal(t, time.Minute, slept)
+	delta += time.Hour
+	slept, _ = w.Sample()
+	assert.Equal(t, time.Hour, slept)
+
+	// An unsupported platform read reports not-ok
+	ok = false
+	_, sok = w.Sample()
+	assert.False(t, sok)
+}
+
+// TestWakeDetectorPlatformClock smoke tests the real clock pair: two samples close together must not report a
+// wake on a machine that isn't suspending mid-test.
+func TestWakeDetectorPlatformClock(t *testing.T) {
+	w := newWakeDetector()
+	if _, ok := w.Sample(); !ok {
+		t.Skip("platform has no suspend clock pair")
+	}
+	slept, ok := w.Sample()
+	assert.True(t, ok)
+	assert.Less(t, slept, time.Second)
+}