udp_linux.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743
  1. //go:build !android && !e2e_testing
  2. // +build !android,!e2e_testing
  3. package udp
  4. import (
  5. "context"
  6. "encoding/binary"
  7. "fmt"
  8. "net"
  9. "net/netip"
  10. "syscall"
  11. "unsafe"
  12. "github.com/rcrowley/go-metrics"
  13. "github.com/sirupsen/logrus"
  14. "github.com/slackhq/nebula/config"
  15. "golang.org/x/sys/unix"
  16. )
  17. type StdConn struct {
  18. udpConn *net.UDPConn
  19. rawConn syscall.RawConn
  20. isV4 bool
  21. l *logrus.Logger
  22. batch int
  23. // sendmmsg scratch. Each queue has its own StdConn, so no locking is
  24. // needed. Sized to MaxWriteBatch at construction; WriteBatch chunks
  25. // larger inputs.
  26. writeMsgs []rawMessage
  27. writeIovs []iovec
  28. writeNames [][]byte
  29. // Per-entry UDP_SEGMENT cmsg scratch. writeCmsg is one contiguous slab
  30. // of MaxWriteBatch * writeCmsgSpace bytes; each entry's cmsg header is
  31. // pre-filled once in prepareWriteMessages. WriteBatch only rewrites the
  32. // 2-byte gso_size payload (and toggles Hdr.Control on/off) per call.
  33. writeCmsg []byte
  34. writeCmsgSpace int
  35. // writeEntryEnd[e] is the bufs index *after* the last packet packed
  36. // into mmsghdr entry e. Used to rewind `i` on partial sendmmsg success.
  37. writeEntryEnd []int
  38. // Preallocated closure + in/out slots for sendmmsg, so the hot path
  39. // does not heap-allocate a fresh closure per call.
  40. writeChunk int
  41. writeSent int
  42. writeErrno syscall.Errno
  43. writeFunc func(fd uintptr) bool
  44. // UDP GSO (sendmsg with UDP_SEGMENT cmsg) support. gsoSupported is
  45. // probed once at socket creation. When true, WriteSegmented takes a
  46. // single-syscall GSO path; otherwise it falls back to a WriteTo loop.
  47. gsoSupported bool
  48. // UDP GRO (recvmsg with UDP_GRO cmsg) support. groSupported is probed
  49. // once at socket creation. When true, listenOutBatch allocates larger
  50. // RX buffers and a per-entry cmsg slot so the kernel can coalesce
  51. // consecutive same-flow datagrams into a single recvmmsg entry; the
  52. // delivered cmsg carries the gso_size used to split them back apart.
  53. groSupported bool
  54. }
  55. func setReusePort(network, address string, c syscall.RawConn) error {
  56. var opErr error
  57. err := c.Control(func(fd uintptr) {
  58. opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
  59. //CloseOnExec already set by the runtime
  60. })
  61. if err != nil {
  62. return err
  63. }
  64. return opErr
  65. }
  66. func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
  67. listen := netip.AddrPortFrom(ip, uint16(port))
  68. lc := net.ListenConfig{}
  69. if multi {
  70. lc.Control = setReusePort
  71. }
  72. //this context is only used during the bind operation, you can't cancel it to kill the socket
  73. pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
  74. if err != nil {
  75. return nil, fmt.Errorf("unable to open socket: %s", err)
  76. }
  77. udpConn := pc.(*net.UDPConn)
  78. rawConn, err := udpConn.SyscallConn()
  79. if err != nil {
  80. _ = udpConn.Close()
  81. return nil, err
  82. }
  83. //gotta find out if we got an AF_INET6 socket or not:
  84. out := &StdConn{
  85. udpConn: udpConn,
  86. rawConn: rawConn,
  87. l: l,
  88. batch: batch,
  89. }
  90. af, err := out.getSockOptInt(unix.SO_DOMAIN)
  91. if err != nil {
  92. _ = out.Close()
  93. return nil, err
  94. }
  95. out.isV4 = af == unix.AF_INET
  96. out.prepareWriteMessages(MaxWriteBatch)
  97. out.writeFunc = out.sendmmsgRawWrite
  98. out.prepareGSO()
  99. // GRO delivers coalesced superpackets that need a cmsg to split back
  100. // into segments. The single-packet RX path uses ReadFromUDPAddrPort
  101. // and cannot see that cmsg, so only enable GRO for the batch path.
  102. if batch > 1 {
  103. out.prepareGRO()
  104. }
  105. return out, nil
  106. }
  107. // prepareWriteMessages allocates one mmsghdr/iovec/sockaddr/cmsg scratch
  108. // slot per sendmmsg entry. The iovec slab is sized to the same n so a
  109. // single entry can fan out to up to n iovecs (needed for UDP_SEGMENT runs
  110. // that coalesce consecutive bufs into one entry). Hdr.Iov / Hdr.Iovlen /
  111. // Hdr.Control / Hdr.Controllen are wired per call since each entry can
  112. // span a variable number of iovecs and may or may not carry a cmsg.
  113. func (u *StdConn) prepareWriteMessages(n int) {
  114. u.writeMsgs = make([]rawMessage, n)
  115. u.writeIovs = make([]iovec, n)
  116. u.writeNames = make([][]byte, n)
  117. u.writeEntryEnd = make([]int, n)
  118. u.writeCmsgSpace = unix.CmsgSpace(2)
  119. u.writeCmsg = make([]byte, n*u.writeCmsgSpace)
  120. for k := 0; k < n; k++ {
  121. off := k * u.writeCmsgSpace
  122. h := (*unix.Cmsghdr)(unsafe.Pointer(&u.writeCmsg[off]))
  123. h.Level = unix.SOL_UDP
  124. h.Type = unix.UDP_SEGMENT
  125. setCmsgLen(h, unix.CmsgLen(2))
  126. }
  127. for i := range u.writeMsgs {
  128. u.writeNames[i] = make([]byte, unix.SizeofSockaddrInet6)
  129. u.writeMsgs[i].Hdr.Name = &u.writeNames[i][0]
  130. }
  131. }
  132. // maxGSOSegments caps the per-sendmsg GSO fan-out. Linux kernels have
  133. // historically capped UDP_MAX_SEGMENTS at 64; newer kernels raise it to 128
  134. // but we stay conservative so the same code works everywhere.
  135. const maxGSOSegments = 64
  136. // maxGSOBytes bounds the total payload per sendmsg() when UDP_SEGMENT is
  137. // set. The kernel stitches all iovecs into a single skb whose length the
  138. // UDP length field can represent, and also enforces sk_gso_max_size (which
  139. // on most devices is 65536). We use 65535 so ciphertext + headers always
  140. // fits, avoiding EMSGSIZE on large TSO superpackets.
  141. const maxGSOBytes = 65535
  142. // prepareGSO probes UDP_SEGMENT support
  143. func (u *StdConn) prepareGSO() {
  144. var probeErr error
  145. if err := u.rawConn.Control(func(fd uintptr) {
  146. probeErr = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_SEGMENT, 0)
  147. }); err != nil {
  148. return
  149. }
  150. if probeErr != nil {
  151. return
  152. }
  153. u.gsoSupported = true
  154. }
  155. // udpGROBufferSize sizes the per-entry recvmmsg buffer when UDP_GRO is on.
  156. // The kernel stitches a run of same-flow datagrams into a single skb whose
  157. // length is bounded by sk_gso_max_size (typically 65535); anything larger
  158. // would be MSG_TRUNCed. We use the maximum representable UDP length so a
  159. // full superpacket always lands intact.
  160. const udpGROBufferSize = 65535
  161. // udpGROCmsgPayload is the size of the UDP_GRO cmsg data delivered by the
  162. // kernel: a single int (gso_size in bytes). See udp_cmsg_recv() in
  163. // net/ipv4/udp.c.
  164. const udpGROCmsgPayload = 4
  165. // prepareGRO turns on UDP_GRO so the kernel coalesces consecutive same-flow
  166. // datagrams into one recvmmsg entry, with a cmsg carrying the gso_size used
  167. // to split them back apart on the application side.
  168. func (u *StdConn) prepareGRO() {
  169. var probeErr error
  170. if err := u.rawConn.Control(func(fd uintptr) {
  171. probeErr = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_GRO, 1)
  172. }); err != nil {
  173. return
  174. }
  175. if probeErr != nil {
  176. return
  177. }
  178. u.groSupported = true
  179. }
  180. func (u *StdConn) SupportsMultipleReaders() bool {
  181. return true
  182. }
  183. func (u *StdConn) Rebind() error {
  184. return nil
  185. }
  186. func (u *StdConn) getSockOptInt(opt int) (int, error) {
  187. if u.rawConn == nil {
  188. return 0, fmt.Errorf("no UDP connection")
  189. }
  190. var out int
  191. var opErr error
  192. err := u.rawConn.Control(func(fd uintptr) {
  193. out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
  194. })
  195. if err != nil {
  196. return 0, err
  197. }
  198. return out, opErr
  199. }
  200. func (u *StdConn) setSockOptInt(opt int, n int) error {
  201. if u.rawConn == nil {
  202. return fmt.Errorf("no UDP connection")
  203. }
  204. var opErr error
  205. err := u.rawConn.Control(func(fd uintptr) {
  206. opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
  207. })
  208. if err != nil {
  209. return err
  210. }
  211. return opErr
  212. }
  213. func (u *StdConn) SetRecvBuffer(n int) error {
  214. return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
  215. }
  216. func (u *StdConn) SetSendBuffer(n int) error {
  217. return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
  218. }
  219. func (u *StdConn) SetSoMark(mark int) error {
  220. return u.setSockOptInt(unix.SO_MARK, mark)
  221. }
  222. func (u *StdConn) GetRecvBuffer() (int, error) {
  223. return u.getSockOptInt(unix.SO_RCVBUF)
  224. }
  225. func (u *StdConn) GetSendBuffer() (int, error) {
  226. return u.getSockOptInt(unix.SO_SNDBUF)
  227. }
  228. func (u *StdConn) GetSoMark() (int, error) {
  229. return u.getSockOptInt(unix.SO_MARK)
  230. }
  231. func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
  232. a := u.udpConn.LocalAddr()
  233. switch v := a.(type) {
  234. case *net.UDPAddr:
  235. addr, ok := netip.AddrFromSlice(v.IP)
  236. if !ok {
  237. return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
  238. }
  239. return netip.AddrPortFrom(addr, uint16(v.Port)), nil
  240. default:
  241. return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
  242. }
  243. }
  244. func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
  245. var errno syscall.Errno
  246. n, _, errno := unix.Syscall6(
  247. unix.SYS_RECVMMSG,
  248. fd,
  249. uintptr(unsafe.Pointer(&msgs[0])),
  250. uintptr(len(msgs)),
  251. unix.MSG_WAITFORONE,
  252. 0,
  253. 0,
  254. )
  255. if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
  256. // No data available, block for I/O and try again.
  257. return int(n), false, nil
  258. }
  259. if errno != 0 {
  260. return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
  261. }
  262. return int(n), true, nil
  263. }
  264. func (u *StdConn) listenOutSingle(r EncReader, flush func()) error {
  265. var err error
  266. var n int
  267. var from netip.AddrPort
  268. buffer := make([]byte, MTU)
  269. for {
  270. n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
  271. if err != nil {
  272. return err
  273. }
  274. from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
  275. r(from, buffer[:n])
  276. flush()
  277. }
  278. }
  279. func (u *StdConn) listenOutBatch(r EncReader, flush func()) error {
  280. var ip netip.Addr
  281. var n int
  282. var operr error
  283. bufSize := MTU
  284. cmsgSpace := 0
  285. if u.groSupported {
  286. bufSize = udpGROBufferSize
  287. cmsgSpace = unix.CmsgSpace(udpGROCmsgPayload)
  288. }
  289. msgs, buffers, names, _ := u.PrepareRawMessages(u.batch, bufSize, cmsgSpace)
  290. //reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
  291. //defining it outside the loop so it gets re-used
  292. reader := func(fd uintptr) (done bool) {
  293. n, done, operr = recvmmsg(fd, msgs)
  294. return done
  295. }
  296. for {
  297. if cmsgSpace > 0 {
  298. for i := range msgs {
  299. setMsgControllen(&msgs[i].Hdr, cmsgSpace)
  300. }
  301. }
  302. err := u.rawConn.Read(reader)
  303. if err != nil {
  304. return err
  305. }
  306. if operr != nil {
  307. return operr
  308. }
  309. for i := 0; i < n; i++ {
  310. // Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
  311. if u.isV4 {
  312. ip, _ = netip.AddrFromSlice(names[i][4:8])
  313. } else {
  314. ip, _ = netip.AddrFromSlice(names[i][8:24])
  315. }
  316. from := netip.AddrPortFrom(ip.Unmap(), binary.BigEndian.Uint16(names[i][2:4]))
  317. payload := buffers[i][:msgs[i].Len]
  318. segSize := 0
  319. if u.groSupported {
  320. segSize = parseUDPGRO(&msgs[i].Hdr)
  321. }
  322. if segSize <= 0 || segSize >= len(payload) {
  323. // No coalescing happened (or a lone datagram).
  324. r(from, payload)
  325. continue
  326. }
  327. // GRO superpacket: the kernel guarantees every segment is
  328. // exactly segSize bytes except for the final one, which may be
  329. // short.
  330. for off := 0; off < len(payload); off += segSize {
  331. end := off + segSize
  332. if end > len(payload) {
  333. end = len(payload)
  334. }
  335. r(from, payload[off:end])
  336. }
  337. }
  338. // End-of-batch: let callers (e.g. TUN write coalescer) flush any
  339. // state they accumulated across this batch.
  340. flush()
  341. }
  342. }
  343. // parseUDPGRO walks the control buffer on hdr looking for a SOL_UDP/UDP_GRO
  344. // cmsg and returns the gso_size (bytes per coalesced segment) it carries.
  345. // Returns 0 when no UDP_GRO cmsg is present, which is the normal case for
  346. // lone datagrams that the kernel did not coalesce.
  347. func parseUDPGRO(hdr *msghdr) int {
  348. controllen := int(hdr.Controllen)
  349. if controllen < unix.SizeofCmsghdr || hdr.Control == nil {
  350. return 0
  351. }
  352. ctrl := unsafe.Slice(hdr.Control, controllen)
  353. off := 0
  354. for off+unix.SizeofCmsghdr <= len(ctrl) {
  355. ch := (*unix.Cmsghdr)(unsafe.Pointer(&ctrl[off]))
  356. clen := int(ch.Len)
  357. if clen < unix.SizeofCmsghdr || off+clen > len(ctrl) {
  358. return 0
  359. }
  360. if ch.Level == unix.SOL_UDP && ch.Type == unix.UDP_GRO {
  361. dataOff := off + unix.CmsgLen(0)
  362. if dataOff+udpGROCmsgPayload <= len(ctrl) {
  363. return int(int32(binary.NativeEndian.Uint32(ctrl[dataOff : dataOff+udpGROCmsgPayload])))
  364. }
  365. return 0
  366. }
  367. // Advance by the aligned cmsg space. CmsgSpace(n) is the stride
  368. // from one header to the next (len aligned up to the platform's
  369. // cmsg alignment).
  370. off += unix.CmsgSpace(clen - unix.CmsgLen(0))
  371. }
  372. return 0
  373. }
  374. func (u *StdConn) ListenOut(r EncReader, flush func()) error {
  375. if u.batch == 1 {
  376. return u.listenOutSingle(r, flush)
  377. } else {
  378. return u.listenOutBatch(r, flush)
  379. }
  380. }
  381. func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
  382. _, err := u.udpConn.WriteToUDPAddrPort(b, ip)
  383. return err
  384. }
  385. // WriteBatch sends bufs via sendmmsg(2) using the preallocated scratch on
  386. // StdConn. Consecutive packets to the same destination with matching segment
  387. // sizes (all but possibly the last) are coalesced into a single mmsghdr entry
  388. // carrying a UDP_SEGMENT cmsg, so one syscall can mix runs of GSO superpackets
  389. // with plain one-off datagrams. Without GSO support every packet is its own
  390. // entry, matching the prior behaviour.
  391. //
  392. // Chunks larger than the scratch are processed across multiple syscalls. If
  393. // sendmmsg returns a fatal error before any entry is sent we fall back to
  394. // per-packet WriteTo for that chunk so the caller still gets best-effort
  395. // delivery.
  396. func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) error {
  397. if len(bufs) != len(addrs) {
  398. return fmt.Errorf("WriteBatch: len(bufs)=%d != len(addrs)=%d", len(bufs), len(addrs))
  399. }
  400. i := 0
  401. for i < len(bufs) {
  402. baseI := i
  403. entry := 0
  404. iovIdx := 0
  405. for entry < len(u.writeMsgs) && i < len(bufs) {
  406. iovBudget := len(u.writeIovs) - iovIdx
  407. if iovBudget < 1 {
  408. break
  409. }
  410. runLen, segSize := u.planRun(bufs, addrs, i, iovBudget)
  411. if runLen == 0 {
  412. break
  413. }
  414. for k := 0; k < runLen; k++ {
  415. b := bufs[i+k]
  416. if len(b) == 0 {
  417. u.writeIovs[iovIdx+k].Base = nil
  418. setIovLen(&u.writeIovs[iovIdx+k], 0)
  419. } else {
  420. u.writeIovs[iovIdx+k].Base = &b[0]
  421. setIovLen(&u.writeIovs[iovIdx+k], len(b))
  422. }
  423. }
  424. nlen, err := writeSockaddr(u.writeNames[entry], addrs[i], u.isV4)
  425. if err != nil {
  426. return err
  427. }
  428. hdr := &u.writeMsgs[entry].Hdr
  429. hdr.Iov = &u.writeIovs[iovIdx]
  430. setMsgIovlen(hdr, runLen)
  431. hdr.Namelen = uint32(nlen)
  432. if runLen >= 2 {
  433. off := entry * u.writeCmsgSpace
  434. dataOff := off + unix.CmsgLen(0)
  435. binary.NativeEndian.PutUint16(u.writeCmsg[dataOff:dataOff+2], uint16(segSize))
  436. hdr.Control = &u.writeCmsg[off]
  437. setMsgControllen(hdr, u.writeCmsgSpace)
  438. } else {
  439. hdr.Control = nil
  440. setMsgControllen(hdr, 0)
  441. }
  442. i += runLen
  443. iovIdx += runLen
  444. u.writeEntryEnd[entry] = i
  445. entry++
  446. }
  447. if entry == 0 {
  448. return fmt.Errorf("sendmmsg: no progress")
  449. }
  450. sent, serr := u.sendmmsg(entry)
  451. if serr != nil && sent <= 0 {
  452. // Nothing went out for this chunk; fall back to WriteTo for each
  453. // packet that was queued this iteration.
  454. for k := baseI; k < i; k++ {
  455. if werr := u.WriteTo(bufs[k], addrs[k]); werr != nil {
  456. return werr
  457. }
  458. }
  459. continue
  460. }
  461. if sent == 0 {
  462. return fmt.Errorf("sendmmsg made no progress")
  463. }
  464. // Rewind i to the end of the last successfully sent entry. For a
  465. // full-success send this leaves i unchanged; for a partial send it
  466. // replays the remainder on the next outer-loop iteration.
  467. i = u.writeEntryEnd[sent-1]
  468. }
  469. return nil
  470. }
  471. // planRun groups consecutive packets starting at `start` that can be sent as
  472. // a single UDP GSO superpacket (one sendmmsg entry with UDP_SEGMENT cmsg).
  473. // A run of length 1 means the entry carries no cmsg and the kernel treats
  474. // it as a plain datagram. Returns the run length and the per-segment size
  475. // (which equals len(bufs[start])). Without GSO support every call returns
  476. // runLen=1.
  477. func (u *StdConn) planRun(bufs [][]byte, addrs []netip.AddrPort, start, iovBudget int) (int, int) {
  478. if start >= len(bufs) || iovBudget < 1 {
  479. return 0, 0
  480. }
  481. segSize := len(bufs[start])
  482. if !u.gsoSupported || segSize == 0 || segSize > maxGSOBytes {
  483. return 1, segSize
  484. }
  485. dst := addrs[start]
  486. maxLen := maxGSOSegments
  487. if iovBudget < maxLen {
  488. maxLen = iovBudget
  489. }
  490. runLen := 1
  491. total := segSize
  492. for runLen < maxLen && start+runLen < len(bufs) {
  493. nextLen := len(bufs[start+runLen])
  494. if nextLen == 0 || nextLen > segSize {
  495. break
  496. }
  497. if addrs[start+runLen] != dst {
  498. break
  499. }
  500. if total+nextLen > maxGSOBytes {
  501. break
  502. }
  503. total += nextLen
  504. runLen++
  505. if nextLen < segSize {
  506. // A short packet must be the last in the run.
  507. break
  508. }
  509. }
  510. return runLen, segSize
  511. }
  512. // sendmmsgRawWrite is the preallocated callback passed to rawConn.Write. It
  513. // reads its input (u.writeChunk) and writes its outputs (u.writeSent,
  514. // u.writeErrno) through StdConn fields so the closure itself does not
  515. // capture per-call locals and therefore does not heap-allocate.
  516. func (u *StdConn) sendmmsgRawWrite(fd uintptr) bool {
  517. r1, _, errno := unix.Syscall6(
  518. unix.SYS_SENDMMSG,
  519. fd,
  520. uintptr(unsafe.Pointer(&u.writeMsgs[0])),
  521. uintptr(u.writeChunk),
  522. 0,
  523. 0,
  524. 0,
  525. )
  526. if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
  527. return false
  528. }
  529. u.writeSent = int(r1)
  530. u.writeErrno = errno
  531. return true
  532. }
  533. func (u *StdConn) sendmmsg(n int) (int, error) {
  534. u.writeChunk = n
  535. u.writeSent = 0
  536. u.writeErrno = 0
  537. if err := u.rawConn.Write(u.writeFunc); err != nil {
  538. return u.writeSent, err
  539. }
  540. if u.writeErrno != 0 {
  541. return u.writeSent, &net.OpError{Op: "sendmmsg", Err: u.writeErrno}
  542. }
  543. return u.writeSent, nil
  544. }
  545. // writeSockaddr encodes addr into buf (which must be at least
  546. // SizeofSockaddrInet6 bytes). Returns the number of bytes used. If isV4 is
  547. // true and addr is not a v4 (or v4-in-v6) address, returns an error.
  548. func writeSockaddr(buf []byte, addr netip.AddrPort, isV4 bool) (int, error) {
  549. ap := addr.Addr().Unmap()
  550. if isV4 {
  551. if !ap.Is4() {
  552. return 0, ErrInvalidIPv6RemoteForSocket
  553. }
  554. // struct sockaddr_in: { sa_family_t(2), in_port_t(2, BE), in_addr(4), zero(8) }
  555. // sa_family is host endian.
  556. binary.NativeEndian.PutUint16(buf[0:2], unix.AF_INET)
  557. binary.BigEndian.PutUint16(buf[2:4], addr.Port())
  558. ip4 := ap.As4()
  559. copy(buf[4:8], ip4[:])
  560. for j := 8; j < 16; j++ {
  561. buf[j] = 0
  562. }
  563. return unix.SizeofSockaddrInet4, nil
  564. }
  565. // struct sockaddr_in6: { sa_family_t(2), in_port_t(2, BE), flowinfo(4), in6_addr(16), scope_id(4) }
  566. binary.NativeEndian.PutUint16(buf[0:2], unix.AF_INET6)
  567. binary.BigEndian.PutUint16(buf[2:4], addr.Port())
  568. binary.NativeEndian.PutUint32(buf[4:8], 0)
  569. ip6 := addr.Addr().As16()
  570. copy(buf[8:24], ip6[:])
  571. binary.NativeEndian.PutUint32(buf[24:28], 0)
  572. return unix.SizeofSockaddrInet6, nil
  573. }
  574. func (u *StdConn) ReloadConfig(c *config.C) {
  575. b := c.GetInt("listen.read_buffer", 0)
  576. if b > 0 {
  577. err := u.SetRecvBuffer(b)
  578. if err == nil {
  579. s, err := u.GetRecvBuffer()
  580. if err == nil {
  581. u.l.WithField("size", s).Info("listen.read_buffer was set")
  582. } else {
  583. u.l.WithError(err).Warn("Failed to get listen.read_buffer")
  584. }
  585. } else {
  586. u.l.WithError(err).Error("Failed to set listen.read_buffer")
  587. }
  588. }
  589. b = c.GetInt("listen.write_buffer", 0)
  590. if b > 0 {
  591. err := u.SetSendBuffer(b)
  592. if err == nil {
  593. s, err := u.GetSendBuffer()
  594. if err == nil {
  595. u.l.WithField("size", s).Info("listen.write_buffer was set")
  596. } else {
  597. u.l.WithError(err).Warn("Failed to get listen.write_buffer")
  598. }
  599. } else {
  600. u.l.WithError(err).Error("Failed to set listen.write_buffer")
  601. }
  602. }
  603. b = c.GetInt("listen.so_mark", 0)
  604. s, err := u.GetSoMark()
  605. if b > 0 || (err == nil && s != 0) {
  606. err := u.SetSoMark(b)
  607. if err == nil {
  608. s, err := u.GetSoMark()
  609. if err == nil {
  610. u.l.WithField("mark", s).Info("listen.so_mark was set")
  611. } else {
  612. u.l.WithError(err).Warn("Failed to get listen.so_mark")
  613. }
  614. } else {
  615. u.l.WithError(err).Error("Failed to set listen.so_mark")
  616. }
  617. }
  618. }
  619. func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
  620. var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
  621. if u.rawConn == nil {
  622. return fmt.Errorf("no UDP connection")
  623. }
  624. var opErr error
  625. err := u.rawConn.Control(func(fd uintptr) {
  626. _, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
  627. if syserr != 0 {
  628. opErr = syserr
  629. }
  630. })
  631. if err != nil {
  632. return err
  633. }
  634. return opErr
  635. }
  636. func (u *StdConn) Close() error {
  637. if u.udpConn != nil {
  638. return u.udpConn.Close()
  639. }
  640. return nil
  641. }
  642. func NewUDPStatsEmitter(udpConns []Conn) func() {
  643. // Check if our kernel supports SO_MEMINFO before registering the gauges
  644. var udpGauges [][unix.SK_MEMINFO_VARS]metrics.Gauge
  645. var meminfo [unix.SK_MEMINFO_VARS]uint32
  646. if err := udpConns[0].(*StdConn).getMemInfo(&meminfo); err == nil {
  647. udpGauges = make([][unix.SK_MEMINFO_VARS]metrics.Gauge, len(udpConns))
  648. for i := range udpConns {
  649. udpGauges[i] = [unix.SK_MEMINFO_VARS]metrics.Gauge{
  650. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rmem_alloc", i), nil),
  651. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.rcvbuf", i), nil),
  652. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_alloc", i), nil),
  653. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.sndbuf", i), nil),
  654. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.fwd_alloc", i), nil),
  655. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.wmem_queued", i), nil),
  656. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.optmem", i), nil),
  657. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.backlog", i), nil),
  658. metrics.GetOrRegisterGauge(fmt.Sprintf("udp.%d.drops", i), nil),
  659. }
  660. }
  661. }
  662. return func() {
  663. for i, gauges := range udpGauges {
  664. if err := udpConns[i].(*StdConn).getMemInfo(&meminfo); err == nil {
  665. for j := 0; j < unix.SK_MEMINFO_VARS; j++ {
  666. gauges[j].Update(int64(meminfo[j]))
  667. }
  668. }
  669. }
  670. }
  671. }