handshake_manager.go 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307
  1. package nebula
  2. import (
  3. "bytes"
  4. "context"
  5. "crypto/rand"
  6. "encoding/binary"
  7. "errors"
  8. "log/slog"
  9. "net/netip"
  10. "slices"
  11. "sync"
  12. "time"
  13. "github.com/rcrowley/go-metrics"
  14. "github.com/slackhq/nebula/cert"
  15. "github.com/slackhq/nebula/handshake"
  16. "github.com/slackhq/nebula/header"
  17. "github.com/slackhq/nebula/udp"
  18. )
  19. const (
  20. DefaultHandshakeTryInterval = time.Millisecond * 100
  21. DefaultHandshakeRetries = 10
  22. DefaultHandshakeTriggerBuffer = 64
  23. DefaultUseRelays = true
  24. // maxCachedPackets is how many unsent packets we'll buffer per pending
  25. // handshake before dropping further ones.
  26. maxCachedPackets = 100
  27. // HandshakePacket map keys mirror the IX protocol stage convention:
  28. // stage 0 = the initiator's first message (and what the responder
  29. // receives, stripped of header)
  30. // stage 2 = the responder's reply
  31. // Other handshake patterns will need new keys when added.
  32. handshakePacketStage0 uint8 = 0
  33. handshakePacketStage2 uint8 = 2
  34. )
  35. var (
  36. defaultHandshakeConfig = HandshakeConfig{
  37. tryInterval: DefaultHandshakeTryInterval,
  38. retries: DefaultHandshakeRetries,
  39. triggerBuffer: DefaultHandshakeTriggerBuffer,
  40. useRelays: DefaultUseRelays,
  41. }
  42. )
  43. type HandshakeConfig struct {
  44. tryInterval time.Duration
  45. retries int64
  46. triggerBuffer int
  47. useRelays bool
  48. messageMetrics *MessageMetrics
  49. }
  50. type HandshakeManager struct {
  51. // Mutex for interacting with the vpnIps and indexes maps
  52. sync.RWMutex
  53. vpnIps map[netip.Addr]*HandshakeHostInfo
  54. indexes map[uint32]*HandshakeHostInfo
  55. mainHostMap *HostMap
  56. lightHouse *LightHouse
  57. outside udp.Conn
  58. config HandshakeConfig
  59. OutboundHandshakeTimer *LockingTimerWheel[netip.Addr]
  60. messageMetrics *MessageMetrics
  61. metricInitiated metrics.Counter
  62. metricTimedOut metrics.Counter
  63. f *Interface
  64. l *slog.Logger
  65. // can be used to trigger outbound handshake for the given vpnIp
  66. trigger chan netip.Addr
  67. }
  68. type HandshakeHostInfo struct {
  69. sync.Mutex
  70. startTime time.Time // Time that we first started trying with this handshake
  71. ready bool // Is the handshake ready
  72. initiatingVersionOverride cert.Version // Should we use a non-default cert version for this handshake?
  73. counter int64 // How many attempts have we made so far
  74. lastRemotes []netip.AddrPort // Remotes that we sent to during the previous attempt
  75. packetStore []*cachedPacket // A set of packets to be transmitted once the handshake completes
  76. hostinfo *HostInfo
  77. machine *handshake.Machine // The handshake state machine, set during stage 0 (initiator) or beginHandshake (responder multi-message)
  78. }
  79. func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, st header.MessageSubType, packet []byte, f packetCallback, m *cachedPacketMetrics) {
  80. if len(hh.packetStore) < maxCachedPackets {
  81. tempPacket := make([]byte, len(packet))
  82. copy(tempPacket, packet)
  83. hh.packetStore = append(hh.packetStore, &cachedPacket{t, st, f, tempPacket})
  84. if l.Enabled(context.Background(), slog.LevelDebug) {
  85. hh.hostinfo.logger(l).Debug("Packet store",
  86. "length", len(hh.packetStore),
  87. "stored", true,
  88. )
  89. }
  90. } else {
  91. m.dropped.Inc(1)
  92. if l.Enabled(context.Background(), slog.LevelDebug) {
  93. hh.hostinfo.logger(l).Debug("Packet store",
  94. "length", len(hh.packetStore),
  95. "stored", false,
  96. )
  97. }
  98. }
  99. }
  100. func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
  101. return &HandshakeManager{
  102. vpnIps: map[netip.Addr]*HandshakeHostInfo{},
  103. indexes: map[uint32]*HandshakeHostInfo{},
  104. mainHostMap: mainHostMap,
  105. lightHouse: lightHouse,
  106. outside: outside,
  107. config: config,
  108. trigger: make(chan netip.Addr, config.triggerBuffer),
  109. OutboundHandshakeTimer: NewLockingTimerWheel[netip.Addr](config.tryInterval, hsTimeout(config.retries, config.tryInterval)),
  110. messageMetrics: config.messageMetrics,
  111. metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
  112. metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
  113. l: l,
  114. }
  115. }
  116. func (hm *HandshakeManager) Run(ctx context.Context) {
  117. clockSource := time.NewTicker(hm.config.tryInterval)
  118. defer clockSource.Stop()
  119. for {
  120. select {
  121. case <-ctx.Done():
  122. return
  123. case vpnIP := <-hm.trigger:
  124. hm.handleOutbound(vpnIP, true)
  125. case now := <-clockSource.C:
  126. hm.NextOutboundHandshakeTimerTick(now)
  127. }
  128. }
  129. }
  130. func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *header.H) {
  131. // Gate on known handshake subtypes. Unknown subtypes (or future ones we
  132. // don't yet support) are dropped here rather than silently routed through
  133. // the IX path. Add a case when introducing a new pattern.
  134. switch h.Subtype {
  135. case header.HandshakeIXPSK0:
  136. // supported
  137. default:
  138. hm.l.Debug("dropping handshake with unsupported subtype",
  139. "from", via, "subtype", h.Subtype)
  140. return
  141. }
  142. // First remote allow list check before we know the vpnIp
  143. if !via.IsRelayed {
  144. if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(via.UdpAddr.Addr()) {
  145. hm.l.Debug("lighthouse.remote_allow_list denied incoming handshake", "from", via)
  146. return
  147. }
  148. }
  149. // First message of a new handshake. The wire format requires RemoteIndex
  150. // to be zero here (the initiator has no responder index to fill in yet),
  151. // and generateIndex never allocates 0, so any non-zero RemoteIndex on a
  152. // stage-1 packet is malformed or someone probing for an index collision.
  153. // Drop without paying the cost of running noise on a pending Machine.
  154. if h.MessageCounter == 1 {
  155. if h.RemoteIndex != 0 {
  156. hm.l.Debug("dropping stage-1 handshake with non-zero RemoteIndex",
  157. "from", via, "remoteIndex", h.RemoteIndex)
  158. return
  159. }
  160. hm.beginHandshake(via, packet, h)
  161. return
  162. }
  163. // Continuation message must match a pending handshake by index.
  164. // Anything else is an orphaned packet (e.g., late retransmit after
  165. // timeout) and is dropped.
  166. if hh := hm.queryIndex(h.RemoteIndex); hh != nil {
  167. hm.continueHandshake(via, hh, packet)
  168. return
  169. }
  170. }
  171. func (hm *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time) {
  172. hm.OutboundHandshakeTimer.Advance(now)
  173. for {
  174. vpnIp, has := hm.OutboundHandshakeTimer.Purge()
  175. if !has {
  176. break
  177. }
  178. hm.handleOutbound(vpnIp, false)
  179. }
  180. }
  181. func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered bool) {
  182. hh := hm.queryVpnIp(vpnIp)
  183. if hh == nil {
  184. return
  185. }
  186. hh.Lock()
  187. defer hh.Unlock()
  188. hostinfo := hh.hostinfo
  189. // If we are out of time, clean up
  190. if hh.counter >= hm.config.retries {
  191. fields := []any{
  192. "udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()),
  193. "initiatorIndex", hh.hostinfo.localIndexId,
  194. "remoteIndex", hh.hostinfo.remoteIndexId,
  195. "durationNs", time.Since(hh.startTime).Nanoseconds(),
  196. }
  197. // hh.machine can be nil here if buildStage0Packet never succeeded
  198. // (e.g., no certificate available). In that case there's no useful
  199. // handshake metadata to log.
  200. if hh.machine != nil {
  201. fields = append(fields, "handshake", m{
  202. "stage": uint64(hh.machine.MessageIndex()),
  203. "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()),
  204. })
  205. }
  206. hh.hostinfo.logger(hm.l).Info("Handshake timed out", fields...)
  207. hm.metricTimedOut.Inc(1)
  208. hm.DeleteHostInfo(hostinfo)
  209. return
  210. }
  211. // Increment the counter to increase our delay, linear backoff
  212. hh.counter++
  213. // Check if we have a handshake packet to transmit yet
  214. if !hh.ready {
  215. if !hm.buildStage0Packet(hh) {
  216. hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
  217. return
  218. }
  219. }
  220. // TODO: this hardcodes "always retransmit stage 0", which is correct for
  221. // IX (the initiator only ever sends one packet, msg1) but wrong the
  222. // moment a 3+ message pattern lands. The retry loop should resend the
  223. // most recent outgoing message, not always stage 0. That implies
  224. // HandshakeHostInfo tracking a single "currentOutbound" packet (bytes +
  225. // header metadata) that gets replaced as the handshake progresses,
  226. // instead of indexing into HandshakePacket.
  227. stage0 := hostinfo.HandshakePacket[handshakePacketStage0]
  228. hsFields := m{
  229. "stage": uint64(hh.machine.MessageIndex()),
  230. "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()),
  231. }
  232. // Get a remotes object if we don't already have one.
  233. // This is mainly to protect us as this should never be the case
  234. // NB ^ This comment doesn't jive. It's how the thing gets initialized.
  235. // It's the common path. Should it update every time, in case a future LH query/queries give us more info?
  236. if hostinfo.remotes == nil {
  237. hostinfo.remotes = hm.lightHouse.QueryCache([]netip.Addr{vpnIp})
  238. }
  239. remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())
  240. remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes)
  241. // We only care about a lighthouse trigger if we have new remotes to send to.
  242. // This is a very specific optimization for a fast lighthouse reply.
  243. if lighthouseTriggered && !remotesHaveChanged {
  244. // If we didn't return here a lighthouse could cause us to aggressively send handshakes
  245. return
  246. }
  247. hh.lastRemotes = remotes
  248. // This will generate a load of queries for hosts with only 1 ip
  249. // (such as ones registered to the lighthouse with only a private IP)
  250. // So we only do it one time after attempting 5 handshakes already.
  251. if len(remotes) <= 1 && hh.counter == 5 {
  252. // If we only have 1 remote it is highly likely our query raced with the other host registered within the lighthouse
  253. // Our vpnIp here has a tunnel with a lighthouse but has yet to send a host update packet there so we only know about
  254. // the learned public ip for them. Query again to short circuit the promotion counter
  255. hm.lightHouse.QueryServer(vpnIp)
  256. }
  257. // Send the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply
  258. var sentTo []netip.AddrPort
  259. hostinfo.remotes.ForEach(hm.mainHostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
  260. hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
  261. err := hm.outside.WriteTo(stage0, addr)
  262. if err != nil {
  263. hostinfo.logger(hm.l).Error("Failed to send handshake message",
  264. "udpAddr", addr,
  265. "initiatorIndex", hostinfo.localIndexId,
  266. "handshake", hsFields,
  267. "error", err,
  268. )
  269. } else {
  270. sentTo = append(sentTo, addr)
  271. }
  272. })
  273. // Don't be too noisy or confusing if we fail to send a handshake - if we don't get through we'll eventually log a timeout,
  274. // so only log when the list of remotes has changed
  275. if remotesHaveChanged {
  276. hostinfo.logger(hm.l).Info("Handshake message sent",
  277. "udpAddrs", sentTo,
  278. "initiatorIndex", hostinfo.localIndexId,
  279. "handshake", hsFields,
  280. )
  281. } else if hm.l.Enabled(context.Background(), slog.LevelDebug) {
  282. hostinfo.logger(hm.l).Debug("Handshake message sent",
  283. "udpAddrs", sentTo,
  284. "initiatorIndex", hostinfo.localIndexId,
  285. "handshake", hsFields,
  286. )
  287. }
  288. if hm.config.useRelays && len(hostinfo.remotes.relays) > 0 {
  289. hostinfo.logger(hm.l).Info("Attempt to relay through hosts", "relays", hostinfo.remotes.relays)
  290. // Send a RelayRequest to all known Relay IP's
  291. for _, relay := range hostinfo.remotes.relays {
  292. // Don't relay through the host I'm trying to connect to
  293. if relay == vpnIp {
  294. continue
  295. }
  296. // Don't relay to myself
  297. if hm.f.myVpnAddrsTable.Contains(relay) {
  298. continue
  299. }
  300. relayHostInfo := hm.mainHostMap.QueryVpnAddr(relay)
  301. if relayHostInfo == nil || !relayHostInfo.remote.IsValid() {
  302. hostinfo.logger(hm.l).Info("Establish tunnel to relay target", "relay", relay.String())
  303. hm.f.Handshake(relay)
  304. continue
  305. }
  306. // Check the relay HostInfo to see if we already established a relay through
  307. existingRelay, ok := relayHostInfo.relayState.QueryRelayForByIp(vpnIp)
  308. if !ok {
  309. // No relays exist or requested yet.
  310. if relayHostInfo.remote.IsValid() {
  311. idx, err := AddRelay(hm.l, relayHostInfo, hm.mainHostMap, vpnIp, nil, TerminalType, Requested)
  312. if err != nil {
  313. hostinfo.logger(hm.l).Info("Failed to add relay to hostmap", "relay", relay.String(), "error", err)
  314. }
  315. m := NebulaControl{
  316. Type: NebulaControl_CreateRelayRequest,
  317. InitiatorRelayIndex: idx,
  318. }
  319. switch relayHostInfo.GetCert().Certificate.Version() {
  320. case cert.Version1:
  321. if !hm.f.myVpnAddrs[0].Is4() {
  322. hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 network because the relay is not running a current nebula version")
  323. continue
  324. }
  325. if !vpnIp.Is4() {
  326. hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 remote network because the relay is not running a current nebula version")
  327. continue
  328. }
  329. b := hm.f.myVpnAddrs[0].As4()
  330. m.OldRelayFromAddr = binary.BigEndian.Uint32(b[:])
  331. b = vpnIp.As4()
  332. m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
  333. case cert.Version2:
  334. m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
  335. m.RelayToAddr = netAddrToProtoAddr(vpnIp)
  336. default:
  337. hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
  338. continue
  339. }
  340. msg, err := m.Marshal()
  341. if err != nil {
  342. hostinfo.logger(hm.l).Error("Failed to marshal Control message to create relay", "error", err)
  343. } else {
  344. hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
  345. hm.l.Info("send CreateRelayRequest",
  346. "relayFrom", hm.f.myVpnAddrs[0],
  347. "relayTo", vpnIp,
  348. "initiatorRelayIndex", idx,
  349. "relay", relay,
  350. )
  351. }
  352. }
  353. continue
  354. }
  355. switch existingRelay.State {
  356. case Established:
  357. hostinfo.logger(hm.l).Info("Send handshake via relay", "relay", relay.String())
  358. hm.f.SendVia(relayHostInfo, existingRelay, hostinfo.HandshakePacket[handshakePacketStage0], make([]byte, 12), make([]byte, mtu), false)
  359. case Disestablished:
  360. // Mark this relay as 'requested'
  361. relayHostInfo.relayState.UpdateRelayForByIpState(vpnIp, Requested)
  362. fallthrough
  363. case Requested:
  364. hostinfo.logger(hm.l).Info("Re-send CreateRelay request", "relay", relay.String())
  365. // Re-send the CreateRelay request, in case the previous one was lost.
  366. m := NebulaControl{
  367. Type: NebulaControl_CreateRelayRequest,
  368. InitiatorRelayIndex: existingRelay.LocalIndex,
  369. }
  370. switch relayHostInfo.GetCert().Certificate.Version() {
  371. case cert.Version1:
  372. if !hm.f.myVpnAddrs[0].Is4() {
  373. hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 network because the relay is not running a current nebula version")
  374. continue
  375. }
  376. if !vpnIp.Is4() {
  377. hostinfo.logger(hm.l).Error("can not establish v1 relay with a v6 remote network because the relay is not running a current nebula version")
  378. continue
  379. }
  380. b := hm.f.myVpnAddrs[0].As4()
  381. m.OldRelayFromAddr = binary.BigEndian.Uint32(b[:])
  382. b = vpnIp.As4()
  383. m.OldRelayToAddr = binary.BigEndian.Uint32(b[:])
  384. case cert.Version2:
  385. m.RelayFromAddr = netAddrToProtoAddr(hm.f.myVpnAddrs[0])
  386. m.RelayToAddr = netAddrToProtoAddr(vpnIp)
  387. default:
  388. hostinfo.logger(hm.l).Error("Unknown certificate version found while creating relay")
  389. continue
  390. }
  391. msg, err := m.Marshal()
  392. if err != nil {
  393. hostinfo.logger(hm.l).Error("Failed to marshal Control message to create relay", "error", err)
  394. } else {
  395. // This must send over the hostinfo, not over hm.Hosts[ip]
  396. hm.f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
  397. hm.l.Info("send CreateRelayRequest",
  398. "relayFrom", hm.f.myVpnAddrs[0],
  399. "relayTo", vpnIp,
  400. "initiatorRelayIndex", existingRelay.LocalIndex,
  401. "relay", relay,
  402. )
  403. }
  404. case PeerRequested:
  405. // PeerRequested only occurs in Forwarding relays, not Terminal relays, and this is a Terminal relay case.
  406. fallthrough
  407. default:
  408. hostinfo.logger(hm.l).Error("Relay unexpected state",
  409. "vpnIp", vpnIp,
  410. "state", existingRelay.State,
  411. "relay", relay,
  412. )
  413. }
  414. }
  415. }
  416. // If a lighthouse triggered this attempt then we are still in the timer wheel and do not need to re-add
  417. if !lighthouseTriggered {
  418. hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
  419. }
  420. }
  421. // GetOrHandshake will try to find a hostinfo with a fully formed tunnel or start a new handshake if one is not present
  422. // The 2nd argument will be true if the hostinfo is ready to transmit traffic
  423. func (hm *HandshakeManager) GetOrHandshake(vpnIp netip.Addr, cacheCb func(*HandshakeHostInfo)) (*HostInfo, bool) {
  424. hm.mainHostMap.RLock()
  425. h, ok := hm.mainHostMap.Hosts[vpnIp]
  426. hm.mainHostMap.RUnlock()
  427. if ok {
  428. // Do not attempt promotion if you are a lighthouse
  429. if !hm.lightHouse.amLighthouse {
  430. h.TryPromoteBest(hm.mainHostMap.GetPreferredRanges(), hm.f)
  431. }
  432. return h, true
  433. }
  434. return hm.StartHandshake(vpnIp, cacheCb), false
  435. }
  436. // StartHandshake will ensure a handshake is currently being attempted for the provided vpn ip
  437. func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*HandshakeHostInfo)) *HostInfo {
  438. hm.Lock()
  439. if hh, ok := hm.vpnIps[vpnAddr]; ok {
  440. // We are already trying to handshake with this vpn ip
  441. if cacheCb != nil {
  442. cacheCb(hh)
  443. }
  444. hm.Unlock()
  445. return hh.hostinfo
  446. }
  447. hostinfo := &HostInfo{
  448. vpnAddrs: []netip.Addr{vpnAddr},
  449. HandshakePacket: make(map[uint8][]byte, 0),
  450. relayState: RelayState{
  451. relays: nil,
  452. relayForByAddr: map[netip.Addr]*Relay{},
  453. relayForByIdx: map[uint32]*Relay{},
  454. },
  455. }
  456. hh := &HandshakeHostInfo{
  457. hostinfo: hostinfo,
  458. startTime: time.Now(),
  459. }
  460. hm.vpnIps[vpnAddr] = hh
  461. hm.metricInitiated.Inc(1)
  462. hm.OutboundHandshakeTimer.Add(vpnAddr, hm.config.tryInterval)
  463. if cacheCb != nil {
  464. cacheCb(hh)
  465. }
  466. // If this is a static host, we don't need to wait for the HostQueryReply
  467. // We can trigger the handshake right now
  468. _, doTrigger := hm.lightHouse.GetStaticHostList()[vpnAddr]
  469. if !doTrigger {
  470. // Add any calculated remotes, and trigger early handshake if one found
  471. doTrigger = hm.lightHouse.addCalculatedRemotes(vpnAddr)
  472. }
  473. if doTrigger {
  474. select {
  475. case hm.trigger <- vpnAddr:
  476. default:
  477. }
  478. }
  479. hm.Unlock()
  480. hm.lightHouse.QueryServer(vpnAddr)
  481. return hostinfo
  482. }
  483. var (
  484. ErrExistingHostInfo = errors.New("existing hostinfo")
  485. ErrAlreadySeen = errors.New("already seen")
  486. ErrLocalIndexCollision = errors.New("local index collision")
  487. )
  488. // CheckAndComplete checks for any conflicts in the main and pending hostmap
  489. // before adding hostinfo to main. If err is nil, it was added. Otherwise err will be:
  490. //
  491. // ErrAlreadySeen if we already have an entry in the hostmap that has seen the
  492. // exact same handshake packet
  493. //
  494. // ErrExistingHostInfo if we already have an entry in the hostmap for this
  495. // VpnIp and the new handshake was older than the one we currently have
  496. //
  497. // ErrLocalIndexCollision if we already have an entry in the main or pending
  498. // hostmap for the hostinfo.localIndexId.
  499. func (hm *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket uint8, f *Interface) (*HostInfo, error) {
  500. hm.mainHostMap.Lock()
  501. defer hm.mainHostMap.Unlock()
  502. hm.Lock()
  503. defer hm.Unlock()
  504. // Check if we already have a tunnel with this vpn ip
  505. existingHostInfo, found := hm.mainHostMap.Hosts[hostinfo.vpnAddrs[0]]
  506. if found && existingHostInfo != nil {
  507. testHostInfo := existingHostInfo
  508. for testHostInfo != nil {
  509. // Is it just a delayed handshake packet?
  510. if bytes.Equal(hostinfo.HandshakePacket[handshakePacket], testHostInfo.HandshakePacket[handshakePacket]) {
  511. return testHostInfo, ErrAlreadySeen
  512. }
  513. testHostInfo = testHostInfo.next
  514. }
  515. // Is this a newer handshake?
  516. if existingHostInfo.lastHandshakeTime >= hostinfo.lastHandshakeTime && !existingHostInfo.ConnectionState.initiator {
  517. return existingHostInfo, ErrExistingHostInfo
  518. }
  519. existingHostInfo.logger(hm.l).Info("Taking new handshake")
  520. }
  521. existingIndex, found := hm.mainHostMap.Indexes[hostinfo.localIndexId]
  522. if found {
  523. // We have a collision, but for a different hostinfo
  524. return existingIndex, ErrLocalIndexCollision
  525. }
  526. existingPendingIndex, found := hm.indexes[hostinfo.localIndexId]
  527. if found && existingPendingIndex.hostinfo != hostinfo {
  528. // We have a collision, but for a different hostinfo
  529. return existingPendingIndex.hostinfo, ErrLocalIndexCollision
  530. }
  531. existingRemoteIndex, found := hm.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId]
  532. if found && existingRemoteIndex != nil && existingRemoteIndex.vpnAddrs[0] != hostinfo.vpnAddrs[0] {
  533. // We have a collision, but this can happen since we can't control
  534. // the remote ID. Just log about the situation as a note.
  535. hostinfo.logger(hm.l).Info("New host shadows existing host remoteIndex",
  536. "remoteIndex", hostinfo.remoteIndexId,
  537. "collision", existingRemoteIndex.vpnAddrs,
  538. )
  539. }
  540. hm.mainHostMap.unlockedAddHostInfo(hostinfo, f)
  541. return existingHostInfo, nil
  542. }
  543. // Complete is a simpler version of CheckAndComplete when we already know we
  544. // won't have a localIndexId collision because we already have an entry in the
  545. // pendingHostMap. An existing hostinfo is returned if there was one.
  546. func (hm *HandshakeManager) Complete(hostinfo *HostInfo, f *Interface) {
  547. hm.mainHostMap.Lock()
  548. defer hm.mainHostMap.Unlock()
  549. hm.Lock()
  550. defer hm.Unlock()
  551. existingRemoteIndex, found := hm.mainHostMap.RemoteIndexes[hostinfo.remoteIndexId]
  552. if found && existingRemoteIndex != nil {
  553. // We have a collision, but this can happen since we can't control
  554. // the remote ID. Just log about the situation as a note.
  555. hostinfo.logger(hm.l).Info("New host shadows existing host remoteIndex",
  556. "remoteIndex", hostinfo.remoteIndexId,
  557. "collision", existingRemoteIndex.vpnAddrs,
  558. )
  559. }
  560. // We need to remove from the pending hostmap first to avoid undoing work when after to the main hostmap.
  561. hm.unlockedDeleteHostInfo(hostinfo)
  562. hm.mainHostMap.unlockedAddHostInfo(hostinfo, f)
  563. }
  564. // allocateIndex generates a unique localIndexId for this HostInfo
  565. // and adds it to the pendingHostMap. Will error if we are unable to generate
  566. // a unique localIndexId
  567. func (hm *HandshakeManager) allocateIndex(hh *HandshakeHostInfo) (uint32, error) {
  568. hm.mainHostMap.RLock()
  569. defer hm.mainHostMap.RUnlock()
  570. hm.Lock()
  571. defer hm.Unlock()
  572. for range 32 {
  573. index, err := generateIndex(hm.l)
  574. if err != nil {
  575. return 0, err
  576. }
  577. _, inPending := hm.indexes[index]
  578. _, inMain := hm.mainHostMap.Indexes[index]
  579. if !inMain && !inPending {
  580. hh.hostinfo.localIndexId = index
  581. hm.indexes[index] = hh
  582. return index, nil
  583. }
  584. }
  585. return 0, errors.New("failed to generate unique localIndexId")
  586. }
  587. func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
  588. hm.Lock()
  589. defer hm.Unlock()
  590. hm.unlockedDeleteHostInfo(hostinfo)
  591. }
  592. func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
  593. for _, addr := range hostinfo.vpnAddrs {
  594. delete(hm.vpnIps, addr)
  595. }
  596. if len(hm.vpnIps) == 0 {
  597. hm.vpnIps = map[netip.Addr]*HandshakeHostInfo{}
  598. }
  599. delete(hm.indexes, hostinfo.localIndexId)
  600. if len(hm.indexes) == 0 {
  601. hm.indexes = map[uint32]*HandshakeHostInfo{}
  602. }
  603. if hm.l.Enabled(context.Background(), slog.LevelDebug) {
  604. hm.l.Debug("Pending hostmap hostInfo deleted",
  605. "hostMap", m{"mapTotalSize": len(hm.vpnIps),
  606. "vpnAddrs": hostinfo.vpnAddrs, "indexNumber": hostinfo.localIndexId, "remoteIndexNumber": hostinfo.remoteIndexId},
  607. )
  608. }
  609. }
  610. func (hm *HandshakeManager) QueryVpnAddr(vpnIp netip.Addr) *HostInfo {
  611. hh := hm.queryVpnIp(vpnIp)
  612. if hh != nil {
  613. return hh.hostinfo
  614. }
  615. return nil
  616. }
  617. func (hm *HandshakeManager) queryVpnIp(vpnIp netip.Addr) *HandshakeHostInfo {
  618. hm.RLock()
  619. defer hm.RUnlock()
  620. return hm.vpnIps[vpnIp]
  621. }
  622. func (hm *HandshakeManager) QueryIndex(index uint32) *HostInfo {
  623. hh := hm.queryIndex(index)
  624. if hh != nil {
  625. return hh.hostinfo
  626. }
  627. return nil
  628. }
  629. func (hm *HandshakeManager) queryIndex(index uint32) *HandshakeHostInfo {
  630. hm.RLock()
  631. defer hm.RUnlock()
  632. return hm.indexes[index]
  633. }
  634. func (hm *HandshakeManager) GetPreferredRanges() []netip.Prefix {
  635. return hm.mainHostMap.GetPreferredRanges()
  636. }
  637. func (hm *HandshakeManager) ForEachVpnAddr(f controlEach) {
  638. hm.RLock()
  639. defer hm.RUnlock()
  640. for _, v := range hm.vpnIps {
  641. f(v.hostinfo)
  642. }
  643. }
  644. func (hm *HandshakeManager) ForEachIndex(f controlEach) {
  645. hm.RLock()
  646. defer hm.RUnlock()
  647. for _, v := range hm.indexes {
  648. f(v.hostinfo)
  649. }
  650. }
  651. func (hm *HandshakeManager) EmitStats() {
  652. hm.RLock()
  653. hostLen := len(hm.vpnIps)
  654. indexLen := len(hm.indexes)
  655. hm.RUnlock()
  656. metrics.GetOrRegisterGauge("hostmap.pending.hosts", nil).Update(int64(hostLen))
  657. metrics.GetOrRegisterGauge("hostmap.pending.indexes", nil).Update(int64(indexLen))
  658. hm.mainHostMap.EmitStats()
  659. }
  660. // Utility functions below
  661. func generateIndex(l *slog.Logger) (uint32, error) {
  662. b := make([]byte, 4)
  663. // Let zero mean we don't know the ID, so don't generate zero
  664. var index uint32
  665. for index == 0 {
  666. _, err := rand.Read(b)
  667. if err != nil {
  668. l.Error("Failed to generate index", "error", err)
  669. return 0, err
  670. }
  671. index = binary.BigEndian.Uint32(b)
  672. }
  673. if l.Enabled(context.Background(), slog.LevelDebug) {
  674. l.Debug("Generated index", "index", index)
  675. }
  676. return index, nil
  677. }
  678. func hsTimeout(tries int64, interval time.Duration) time.Duration {
  679. return time.Duration(tries / 2 * ((2 * int64(interval)) + (tries-1)*int64(interval)))
  680. }
  681. // buildStage0Packet creates the initial handshake packet for the initiator.
  682. func (hm *HandshakeManager) buildStage0Packet(hh *HandshakeHostInfo) bool {
  683. cs := hm.f.pki.getCertState()
  684. v := cs.DefaultVersion()
  685. if hh.initiatingVersionOverride != cert.VersionPre1 {
  686. v = hh.initiatingVersionOverride
  687. } else if v < cert.Version2 {
  688. for _, a := range hh.hostinfo.vpnAddrs {
  689. if a.Is6() {
  690. v = cert.Version2
  691. break
  692. }
  693. }
  694. }
  695. cred := cs.GetCredential(v)
  696. if cred == nil {
  697. hm.f.l.Error("Unable to handshake with host because no certificate is available",
  698. "vpnAddrs", hh.hostinfo.vpnAddrs, "certVersion", v)
  699. return false
  700. }
  701. machine, err := handshake.NewMachine(
  702. v, cs.GetCredential,
  703. hm.certVerifier(), func() (uint32, error) { return hm.allocateIndex(hh) },
  704. true, header.HandshakeIXPSK0,
  705. )
  706. if err != nil {
  707. hm.f.l.Error("Failed to create handshake machine",
  708. "vpnAddrs", hh.hostinfo.vpnAddrs, "error", err)
  709. return false
  710. }
  711. msg, err := machine.Initiate(nil)
  712. if err != nil {
  713. hm.f.l.Error("Failed to initiate handshake",
  714. "vpnAddrs", hh.hostinfo.vpnAddrs, "error", err)
  715. return false
  716. }
  717. // hostinfo.ConnectionState stays nil until the handshake completes in
  718. // continueHandshake. Pre-completion control surfaces guard with nil
  719. // checks; the data plane never observes a pending hostinfo.
  720. hh.hostinfo.HandshakePacket[handshakePacketStage0] = msg
  721. hh.machine = machine
  722. hh.ready = true
  723. return true
  724. }
  725. // beginHandshake handles an incoming handshake packet that doesn't match any
  726. // existing pending handshake. It creates a new responder Machine and processes
  727. // the first message.
  728. func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *header.H) {
  729. f := hm.f
  730. cs := f.pki.getCertState()
  731. v := cs.DefaultVersion()
  732. if cs.GetCredential(v) == nil {
  733. f.l.Error("Unable to handshake with host because no certificate is available",
  734. "from", via, "certVersion", v)
  735. return
  736. }
  737. machine, err := handshake.NewMachine(
  738. v, cs.GetCredential,
  739. hm.certVerifier(), func() (uint32, error) { return generateIndex(f.l) },
  740. false, header.HandshakeIXPSK0,
  741. )
  742. if err != nil {
  743. f.l.Error("Failed to create handshake machine", "from", via, "error", err)
  744. return
  745. }
  746. response, result, err := machine.ProcessPacket(nil, packet)
  747. if err != nil {
  748. f.l.Error("Failed to process handshake packet", "from", via, "error", err)
  749. return
  750. }
  751. if result == nil {
  752. // Multi-message pattern: the responder Machine would need to be
  753. // registered in hm.indexes so a future inbound packet finds it via
  754. // continueHandshake. The current manager doesn't do that yet, so
  755. // fail loudly rather than silently dropping the in-flight handshake.
  756. // TODO: support multi-message responder flows (XX, pqIX, etc.).
  757. // See also the IX-shaped cipher key assignment in handshake.Machine.
  758. f.l.Error("multi-message handshake responder is not supported",
  759. "from", via, "error", handshake.ErrMultiMessageUnsupported)
  760. return
  761. }
  762. remoteCert := result.RemoteCert
  763. if remoteCert == nil {
  764. f.l.Error("Handshake did not produce a peer certificate", "from", via)
  765. return
  766. }
  767. // Validate peer identity
  768. vpnAddrs, anyVpnAddrsInCommon, ok := hm.validatePeerCert(via, remoteCert)
  769. if !ok {
  770. return
  771. }
  772. hostinfo := &HostInfo{
  773. ConnectionState: newConnectionStateFromResult(result),
  774. localIndexId: result.LocalIndex,
  775. remoteIndexId: result.RemoteIndex,
  776. vpnAddrs: vpnAddrs,
  777. HandshakePacket: make(map[uint8][]byte, 0),
  778. lastHandshakeTime: result.HandshakeTime,
  779. relayState: RelayState{
  780. relays: nil,
  781. relayForByAddr: map[netip.Addr]*Relay{},
  782. relayForByIdx: map[uint32]*Relay{},
  783. },
  784. }
  785. msg := "Handshake message received"
  786. if !anyVpnAddrsInCommon {
  787. msg = "Handshake message received, but no vpnNetworks in common."
  788. }
  789. f.l.Info(msg,
  790. "vpnAddrs", vpnAddrs,
  791. "from", via,
  792. "certName", remoteCert.Certificate.Name(),
  793. "certVersion", remoteCert.Certificate.Version(),
  794. "fingerprint", remoteCert.Fingerprint,
  795. "issuer", remoteCert.Certificate.Issuer(),
  796. "initiatorIndex", result.RemoteIndex,
  797. "responderIndex", result.LocalIndex,
  798. "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())},
  799. )
  800. // packet aliases the listener's incoming buffer, so this copy must stay.
  801. hostinfo.HandshakePacket[handshakePacketStage0] = make([]byte, len(packet[header.Len:]))
  802. copy(hostinfo.HandshakePacket[handshakePacketStage0], packet[header.Len:])
  803. // response was freshly allocated by ProcessPacket; safe to retain directly.
  804. if response != nil {
  805. hostinfo.HandshakePacket[handshakePacketStage2] = response
  806. }
  807. hostinfo.remotes = f.lightHouse.QueryCache(vpnAddrs)
  808. if !via.IsRelayed {
  809. hostinfo.SetRemote(via.UdpAddr)
  810. }
  811. hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
  812. existing, err := hm.CheckAndComplete(hostinfo, handshakePacketStage0, f)
  813. if err != nil {
  814. hm.handleCheckAndCompleteError(err, existing, hostinfo, via)
  815. return
  816. }
  817. hm.sendHandshakeResponse(via, response, hostinfo, false)
  818. f.connectionManager.AddTrafficWatch(hostinfo)
  819. hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
  820. // Don't wait for UpdateWorker
  821. if f.lightHouse.IsAnyLighthouseAddr(vpnAddrs) {
  822. f.lightHouse.TriggerUpdate()
  823. }
  824. }
  825. // continueHandshake feeds an incoming packet to an existing pending handshake Machine.
  826. func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostInfo, packet []byte) {
  827. f := hm.f
  828. hh.Lock()
  829. defer hh.Unlock()
  830. // Re-verify hh is still tracked. Between queryIndex returning and us taking
  831. // hh.Lock, handleOutbound may have timed out and deleted it. Once we hold
  832. // hh.Lock no other deleter can race our index: handleOutbound also takes
  833. // hh.Lock first, and handleRecvError targets a main-hostmap entry with a
  834. // different localIndexId.
  835. hm.RLock()
  836. cur, ok := hm.indexes[hh.hostinfo.localIndexId]
  837. hm.RUnlock()
  838. if !ok || cur != hh {
  839. return
  840. }
  841. hostinfo := hh.hostinfo
  842. if !via.IsRelayed {
  843. if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
  844. f.l.Debug("lighthouse.remote_allow_list denied incoming handshake",
  845. "vpnAddrs", hostinfo.vpnAddrs, "from", via)
  846. return
  847. }
  848. }
  849. machine := hh.machine
  850. if machine == nil {
  851. f.l.Error("No handshake machine available for continuation",
  852. "vpnAddrs", hostinfo.vpnAddrs, "from", via)
  853. hm.DeleteHostInfo(hostinfo)
  854. return
  855. }
  856. response, result, err := machine.ProcessPacket(nil, packet)
  857. if err != nil {
  858. // Recoverable errors are routine noise, log at Debug. Fatal errors get a Warn.
  859. if machine.Failed() {
  860. f.l.Warn("Failed to process handshake packet, abandoning",
  861. "vpnAddrs", hostinfo.vpnAddrs, "from", via, "error", err)
  862. hm.DeleteHostInfo(hostinfo)
  863. } else {
  864. f.l.Debug("Failed to process handshake packet",
  865. "vpnAddrs", hostinfo.vpnAddrs, "from", via, "error", err)
  866. }
  867. return
  868. }
  869. if response != nil {
  870. hm.sendHandshakeResponse(via, response, hostinfo, false)
  871. }
  872. if result == nil {
  873. return
  874. }
  875. // Handshake complete; build the ConnectionState now that we have keys and a verified peer cert.
  876. hostinfo.ConnectionState = newConnectionStateFromResult(result)
  877. remoteCert := result.RemoteCert
  878. if remoteCert == nil {
  879. f.l.Error("Handshake completed without peer certificate",
  880. "vpnAddrs", hostinfo.vpnAddrs, "from", via)
  881. hm.DeleteHostInfo(hostinfo)
  882. return
  883. }
  884. vpnNetworks := remoteCert.Certificate.Networks()
  885. hostinfo.remoteIndexId = result.RemoteIndex
  886. hostinfo.lastHandshakeTime = result.HandshakeTime
  887. if !via.IsRelayed {
  888. hostinfo.SetRemote(via.UdpAddr)
  889. } else {
  890. hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
  891. }
  892. // Verify correct host responded (initiator check)
  893. vpnAddrs := make([]netip.Addr, len(vpnNetworks))
  894. correctHostResponded := false
  895. anyVpnAddrsInCommon := false
  896. for i, network := range vpnNetworks {
  897. // inside.go drops self-routed packets at the firewall stage, but we'd
  898. // rather not let a self-handshake complete in the first place: it
  899. // wastes a hostmap slot, suppresses no log, and obscures routing
  900. // misconfig. Explicit refusal here mirrors the responder-side check
  901. // in validatePeerCert.
  902. if f.myVpnAddrsTable.Contains(network.Addr()) {
  903. f.l.Error("Refusing to handshake with myself",
  904. "vpnNetworks", vpnNetworks,
  905. "from", via,
  906. "certName", remoteCert.Certificate.Name(),
  907. "certVersion", remoteCert.Certificate.Version(),
  908. "fingerprint", remoteCert.Fingerprint,
  909. "issuer", remoteCert.Certificate.Issuer(),
  910. "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())},
  911. )
  912. hm.DeleteHostInfo(hostinfo)
  913. return
  914. }
  915. vpnAddrs[i] = network.Addr()
  916. if hostinfo.vpnAddrs[0] == network.Addr() {
  917. correctHostResponded = true
  918. }
  919. if f.myVpnNetworksTable.Contains(network.Addr()) {
  920. anyVpnAddrsInCommon = true
  921. }
  922. }
  923. if !correctHostResponded {
  924. f.l.Info("Incorrect host responded to handshake",
  925. "intendedVpnAddrs", hostinfo.vpnAddrs,
  926. "haveVpnNetworks", vpnNetworks,
  927. "from", via,
  928. "certName", remoteCert.Certificate.Name(),
  929. "certVersion", remoteCert.Certificate.Version(),
  930. "fingerprint", remoteCert.Fingerprint,
  931. "issuer", remoteCert.Certificate.Issuer(),
  932. "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())},
  933. )
  934. hm.DeleteHostInfo(hostinfo)
  935. hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
  936. newHH.hostinfo.remotes = hostinfo.remotes
  937. newHH.hostinfo.remotes.BlockRemote(via)
  938. newHH.packetStore = hh.packetStore
  939. hh.packetStore = []*cachedPacket{}
  940. hostinfo.vpnAddrs = vpnAddrs
  941. f.sendCloseTunnel(hostinfo)
  942. })
  943. return
  944. }
  945. duration := time.Since(hh.startTime).Nanoseconds()
  946. msg := "Handshake message received"
  947. if !anyVpnAddrsInCommon {
  948. msg = "Handshake message received, but no vpnNetworks in common."
  949. }
  950. f.l.Info(msg,
  951. "vpnAddrs", vpnAddrs,
  952. "from", via,
  953. "certName", remoteCert.Certificate.Name(),
  954. "certVersion", remoteCert.Certificate.Version(),
  955. "fingerprint", remoteCert.Fingerprint,
  956. "issuer", remoteCert.Certificate.Issuer(),
  957. "initiatorIndex", result.LocalIndex,
  958. "responderIndex", result.RemoteIndex,
  959. "handshake", m{"stage": uint64(machine.MessageIndex()), "style": header.SubTypeName(header.Handshake, machine.Subtype())},
  960. "durationNs", duration,
  961. "sentCachedPackets", len(hh.packetStore),
  962. )
  963. hostinfo.vpnAddrs = vpnAddrs
  964. hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
  965. hm.Complete(hostinfo, f)
  966. f.connectionManager.AddTrafficWatch(hostinfo)
  967. if len(hh.packetStore) > 0 {
  968. if f.l.Enabled(context.Background(), slog.LevelDebug) {
  969. hostinfo.logger(f.l).Debug("Sending stored packets", "count", len(hh.packetStore))
  970. }
  971. nb := make([]byte, 12, 12)
  972. out := make([]byte, mtu)
  973. for _, cp := range hh.packetStore {
  974. cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, nb, out)
  975. }
  976. f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore)))
  977. }
  978. hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
  979. f.metricHandshakes.Update(duration)
  980. // Don't wait for UpdateWorker
  981. if f.lightHouse.IsAnyLighthouseAddr(vpnAddrs) {
  982. f.lightHouse.TriggerUpdate()
  983. }
  984. }
  985. // validatePeerCert checks the peer certificate for self-connection and remote allow list.
  986. // Returns the VPN addrs, whether any of them fall within one of our own VPN
  987. // networks, and true if valid; false if rejected.
  988. func (hm *HandshakeManager) validatePeerCert(via ViaSender, remoteCert *cert.CachedCertificate) ([]netip.Addr, bool, bool) {
  989. f := hm.f
  990. vpnNetworks := remoteCert.Certificate.Networks()
  991. // The cert package rejects host certs with no networks at parse time, so
  992. // reaching this state would mean an invariant was bypassed elsewhere.
  993. // Refuse explicitly so downstream code (which indexes vpnAddrs[0]) can't
  994. // panic if that invariant ever changes.
  995. if len(vpnNetworks) == 0 {
  996. f.l.Info("No networks in certificate",
  997. "from", via, "cert", remoteCert)
  998. return nil, false, false
  999. }
  1000. vpnAddrs := make([]netip.Addr, len(vpnNetworks))
  1001. anyVpnAddrsInCommon := false
  1002. for i, network := range vpnNetworks {
  1003. if f.myVpnAddrsTable.Contains(network.Addr()) {
  1004. f.l.Error("Refusing to handshake with myself",
  1005. "vpnNetworks", vpnNetworks,
  1006. "from", via,
  1007. "certName", remoteCert.Certificate.Name(),
  1008. "certVersion", remoteCert.Certificate.Version(),
  1009. "fingerprint", remoteCert.Fingerprint,
  1010. "issuer", remoteCert.Certificate.Issuer(),
  1011. )
  1012. return nil, false, false
  1013. }
  1014. vpnAddrs[i] = network.Addr()
  1015. if f.myVpnNetworksTable.Contains(network.Addr()) {
  1016. anyVpnAddrsInCommon = true
  1017. }
  1018. }
  1019. if !via.IsRelayed {
  1020. if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, via.UdpAddr.Addr()) {
  1021. f.l.Debug("lighthouse.remote_allow_list denied incoming handshake",
  1022. "vpnAddrs", vpnAddrs, "from", via)
  1023. return nil, false, false
  1024. }
  1025. }
  1026. return vpnAddrs, anyVpnAddrsInCommon, true
  1027. }
  1028. // sendHandshakeResponse sends a handshake response via the appropriate transport.
  1029. // cached is true when msg is a stored response being retransmitted because
  1030. // the peer's stage-1 retransmit landed (the ErrAlreadySeen path); false on a
  1031. // fresh response.
  1032. func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hostinfo *HostInfo, cached bool) {
  1033. if msg == nil {
  1034. return
  1035. }
  1036. f := hm.f
  1037. f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
  1038. // Common log fields. peerCert may be nil during intermediate
  1039. // multi-message flows (handshake hasn't completed yet); skip the cert
  1040. // block if so.
  1041. logFields := []any{
  1042. "vpnAddrs", hostinfo.vpnAddrs,
  1043. "handshake", m{"stage": uint64(2), "style": header.SubTypeName(header.Handshake, header.HandshakeIXPSK0)},
  1044. "cached", cached,
  1045. "initiatorIndex", hostinfo.remoteIndexId,
  1046. "responderIndex", hostinfo.localIndexId,
  1047. }
  1048. if peerCert := hostinfo.ConnectionState.peerCert; peerCert != nil {
  1049. logFields = append(logFields,
  1050. "certName", peerCert.Certificate.Name(),
  1051. "certVersion", peerCert.Certificate.Version(),
  1052. "fingerprint", peerCert.Fingerprint,
  1053. "issuer", peerCert.Certificate.Issuer(),
  1054. )
  1055. }
  1056. if !via.IsRelayed {
  1057. fields := append(logFields, "from", via)
  1058. err := f.outside.WriteTo(msg, via.UdpAddr)
  1059. if err != nil {
  1060. f.l.Error("Failed to send handshake message", append(fields, "error", err)...)
  1061. } else {
  1062. f.l.Info("Handshake message sent", fields...)
  1063. }
  1064. } else {
  1065. if via.relay == nil {
  1066. f.l.Error("Handshake send failed: both addr and via.relay are nil.")
  1067. return
  1068. }
  1069. hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
  1070. // We received a valid handshake on this relay, so make sure the relay
  1071. // state reflects that, in case it had been marked Disestablished.
  1072. via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
  1073. f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
  1074. f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
  1075. }
  1076. }
  1077. // handleCheckAndCompleteError handles errors from CheckAndComplete.
  1078. // This only fires from the responder-side beginHandshake path, after the
  1079. // peer cert has been validated and ConnectionState populated, so peerCert
  1080. // is always non-nil for the cases that log it.
  1081. func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hostinfo *HostInfo, via ViaSender) {
  1082. f := hm.f
  1083. peerCert := hostinfo.ConnectionState.peerCert
  1084. hsFields := m{"stage": uint64(1), "style": header.SubTypeName(header.Handshake, header.HandshakeIXPSK0)}
  1085. switch err {
  1086. case ErrAlreadySeen:
  1087. if existing.SetRemoteIfPreferred(f.hostMap, via) {
  1088. f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
  1089. }
  1090. // Resend the original response. The peer is committed to that response's
  1091. // ephemeral keys; a freshly-built one would have different keys and break
  1092. // the tunnel even though both sides "completed" the handshake.
  1093. if msg := existing.HandshakePacket[handshakePacketStage2]; msg != nil {
  1094. hm.sendHandshakeResponse(via, msg, existing, true)
  1095. }
  1096. case ErrExistingHostInfo:
  1097. f.l.Info("Handshake too old",
  1098. "vpnAddrs", hostinfo.vpnAddrs,
  1099. "from", via,
  1100. "certName", peerCert.Certificate.Name(),
  1101. "certVersion", peerCert.Certificate.Version(),
  1102. "fingerprint", peerCert.Fingerprint,
  1103. "issuer", peerCert.Certificate.Issuer(),
  1104. "oldHandshakeTime", existing.lastHandshakeTime,
  1105. "newHandshakeTime", hostinfo.lastHandshakeTime,
  1106. "initiatorIndex", hostinfo.remoteIndexId,
  1107. "responderIndex", hostinfo.localIndexId,
  1108. "handshake", hsFields,
  1109. )
  1110. f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
  1111. case ErrLocalIndexCollision:
  1112. f.l.Error("Failed to add HostInfo due to localIndex collision",
  1113. "vpnAddrs", hostinfo.vpnAddrs,
  1114. "from", via,
  1115. "certName", peerCert.Certificate.Name(),
  1116. "certVersion", peerCert.Certificate.Version(),
  1117. "fingerprint", peerCert.Fingerprint,
  1118. "issuer", peerCert.Certificate.Issuer(),
  1119. "localIndex", hostinfo.localIndexId,
  1120. "initiatorIndex", hostinfo.remoteIndexId,
  1121. "responderIndex", hostinfo.localIndexId,
  1122. "handshake", hsFields,
  1123. )
  1124. default:
  1125. f.l.Error("Failed to add HostInfo to HostMap",
  1126. "vpnAddrs", hostinfo.vpnAddrs,
  1127. "from", via,
  1128. "error", err,
  1129. "certName", peerCert.Certificate.Name(),
  1130. "certVersion", peerCert.Certificate.Version(),
  1131. "fingerprint", peerCert.Fingerprint,
  1132. "issuer", peerCert.Certificate.Issuer(),
  1133. "initiatorIndex", hostinfo.remoteIndexId,
  1134. "responderIndex", hostinfo.localIndexId,
  1135. "handshake", hsFields,
  1136. )
  1137. }
  1138. }
  1139. // certVerifier returns a CertVerifier that validates certs against the current CA pool.
  1140. func (hm *HandshakeManager) certVerifier() handshake.CertVerifier {
  1141. return func(c cert.Certificate) (*cert.CachedCertificate, error) {
  1142. return hm.f.pki.GetCAPool().VerifyCertificate(time.Now(), c)
  1143. }
  1144. }