|
|
@@ -5,11 +5,9 @@ package udp
|
|
|
|
|
|
import (
|
|
|
"encoding/binary"
|
|
|
- "errors"
|
|
|
"fmt"
|
|
|
"net"
|
|
|
"net/netip"
|
|
|
- "sync"
|
|
|
"syscall"
|
|
|
"time"
|
|
|
"unsafe"
|
|
|
@@ -21,128 +19,8 @@ import (
|
|
|
"golang.org/x/sys/unix"
|
|
|
)
|
|
|
|
|
|
-const (
|
|
|
- defaultGSOMaxSegments = 16
|
|
|
- defaultGSOFlushTimeout = 150 * time.Microsecond
|
|
|
- maxGSOBatchBytes = 0xFFFF
|
|
|
-)
|
|
|
-
|
|
|
-var (
|
|
|
- errGSOFallback = errors.New("udp gso fallback")
|
|
|
- errGSODisabled = errors.New("udp gso disabled")
|
|
|
-)
|
|
|
-
|
|
|
var readTimeout = unix.NsecToTimeval(int64(time.Millisecond * 500))
|
|
|
|
|
|
-type gsoState struct {
|
|
|
- m sync.Mutex
|
|
|
- Buf []byte
|
|
|
- Addr netip.AddrPort
|
|
|
- SegSize int
|
|
|
- MaxSegments int
|
|
|
- MaxBytes int
|
|
|
- FlushTimeout time.Duration
|
|
|
- Timer *time.Timer
|
|
|
-
|
|
|
- packets []*packet.Packet
|
|
|
- msg rawMessage
|
|
|
- name [unix.SizeofSockaddrInet6]byte
|
|
|
- iov []iovec
|
|
|
- ctrl []byte
|
|
|
-}
|
|
|
-
|
|
|
-func (g *gsoState) Init() {
|
|
|
- g.iov = make([]iovec, g.MaxSegments)
|
|
|
- for i := 0; i < g.MaxSegments; i++ {
|
|
|
- g.iov[i] = iovec{}
|
|
|
- }
|
|
|
- g.msg.Hdr.Iov = &g.iov[0]
|
|
|
- g.msg.Hdr.Iovlen = 1
|
|
|
-
|
|
|
- g.packets = make([]*packet.Packet, 0, g.MaxSegments)
|
|
|
- g.ctrl = make([]byte, unix.CmsgSpace(2))
|
|
|
- hdr := (*unix.Cmsghdr)(unsafe.Pointer(&g.ctrl[0]))
|
|
|
- hdr.Level = unix.SOL_UDP
|
|
|
- hdr.Type = unix.UDP_SEGMENT
|
|
|
- setCmsgLen(hdr, unix.CmsgLen(2))
|
|
|
- g.msg.Hdr.Control = &g.ctrl[0]
|
|
|
- g.msg.Hdr.Controllen = uint64(len(g.ctrl))
|
|
|
-
|
|
|
- g.name = [unix.SizeofSockaddrInet6]byte{}
|
|
|
- g.msg.Hdr.Name = &g.name[0]
|
|
|
-
|
|
|
-}
|
|
|
-
|
|
|
-func (g *gsoState) setSegSizeLocked(segSize int) {
|
|
|
- g.SegSize = segSize
|
|
|
- x := unix.CmsgLen(0)
|
|
|
- binary.LittleEndian.PutUint16(g.ctrl[x:x+2], uint16(segSize))
|
|
|
-}
|
|
|
-
|
|
|
-func (g *gsoState) setNameLocked(x netip.AddrPort, isV4 bool) {
|
|
|
- g.Addr = x
|
|
|
- nameLen := encodeSockaddr(g.name[:], g.Addr, isV4)
|
|
|
- g.msg.Hdr.Name = &g.name[0]
|
|
|
- g.msg.Hdr.Namelen = nameLen
|
|
|
-}
|
|
|
-
|
|
|
-func encodeSockaddr(dst []byte, addr netip.AddrPort, isV4 bool) uint32 {
|
|
|
- if isV4 {
|
|
|
- //todo?
|
|
|
- //if !addr.Addr().Is4() {
|
|
|
- // return 0, fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
|
|
|
- //}
|
|
|
- var sa unix.RawSockaddrInet4
|
|
|
- sa.Family = unix.AF_INET
|
|
|
- sa.Addr = addr.Addr().As4()
|
|
|
- binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
|
|
|
- size := unix.SizeofSockaddrInet4
|
|
|
- copy(dst[:size], (*(*[unix.SizeofSockaddrInet4]byte)(unsafe.Pointer(&sa)))[:])
|
|
|
- return uint32(size)
|
|
|
- }
|
|
|
-
|
|
|
- var sa unix.RawSockaddrInet6
|
|
|
- sa.Family = unix.AF_INET6
|
|
|
- sa.Addr = addr.Addr().As16()
|
|
|
- binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], addr.Port())
|
|
|
- size := unix.SizeofSockaddrInet6
|
|
|
- copy(dst[:size], (*(*[unix.SizeofSockaddrInet6]byte)(unsafe.Pointer(&sa)))[:])
|
|
|
- return uint32(size)
|
|
|
-}
|
|
|
-
|
|
|
-func (g *gsoState) sendmsgLocked(fd int) error {
|
|
|
- //name already set
|
|
|
- //ctrl already set
|
|
|
- //g.iov = g.iov[:0]
|
|
|
- g.msg.Hdr.Iovlen = uint64(len(g.packets))
|
|
|
- for i := range g.packets {
|
|
|
- g.iov[i].Base = &g.packets[i].Payload[0]
|
|
|
- g.iov[i].Len = uint64(len(g.packets[i].Payload))
|
|
|
- }
|
|
|
-
|
|
|
- const flags = 0
|
|
|
- for {
|
|
|
- _, _, err := unix.Syscall(
|
|
|
- unix.SYS_SENDMSG,
|
|
|
- uintptr(fd),
|
|
|
- uintptr(unsafe.Pointer(&g.msg)),
|
|
|
- uintptr(flags),
|
|
|
- )
|
|
|
- //todo no matter what, reset things
|
|
|
- for i := range g.packets {
|
|
|
- pool := packet.GetPool()
|
|
|
- pool.Put(g.packets[i])
|
|
|
- }
|
|
|
- g.packets = g.packets[:0]
|
|
|
-
|
|
|
- if err != 0 {
|
|
|
- return &net.OpError{Op: "sendmsg", Err: err}
|
|
|
- }
|
|
|
-
|
|
|
- return nil
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
type StdConn struct {
|
|
|
sysFd int
|
|
|
isV4 bool
|
|
|
@@ -150,7 +28,7 @@ type StdConn struct {
|
|
|
batch int
|
|
|
enableGRO bool
|
|
|
enableGSO bool
|
|
|
- gso gsoState
|
|
|
+ //gso gsoState
|
|
|
}
|
|
|
|
|
|
func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
|
|
|
@@ -358,123 +236,11 @@ func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-func (u *StdConn) WriteTo(p *packet.Packet) error {
|
|
|
- if u.enableGSO && p.Addr.IsValid() {
|
|
|
- if err := u.queueGSOPacket(p); err == nil {
|
|
|
- return nil
|
|
|
- } else if !errors.Is(err, errGSOFallback) {
|
|
|
- return err
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- var err error
|
|
|
+func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
|
|
|
if u.isV4 {
|
|
|
- err = u.writeTo4(p.Payload, p.Addr)
|
|
|
- } else {
|
|
|
- err = u.writeTo4(p.Payload, p.Addr)
|
|
|
- }
|
|
|
- packet.GetPool().Put(p)
|
|
|
- return err
|
|
|
-}
|
|
|
-
|
|
|
-func (u *StdConn) WriteDirect(b []byte, addr netip.AddrPort) error {
|
|
|
- if u.isV4 {
|
|
|
- return u.writeTo4(b, addr)
|
|
|
- }
|
|
|
- return u.writeTo6(b, addr)
|
|
|
-}
|
|
|
-
|
|
|
-func (u *StdConn) scheduleGSOFlushLocked() {
|
|
|
- if u.gso.Timer == nil {
|
|
|
- u.gso.Timer = time.AfterFunc(u.gso.FlushTimeout, u.gsoFlushTimer)
|
|
|
- return
|
|
|
+ return u.writeTo4(b, ip)
|
|
|
}
|
|
|
- u.gso.Timer.Reset(u.gso.FlushTimeout)
|
|
|
-}
|
|
|
-
|
|
|
-func (u *StdConn) stopGSOTimerLocked() {
|
|
|
- if u.gso.Timer != nil {
|
|
|
- u.gso.Timer.Stop()
|
|
|
- u.gso.Timer = nil //todo I also don't like this
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-func (u *StdConn) queueGSOPacket(p *packet.Packet) error {
|
|
|
- if len(p.Payload) == 0 {
|
|
|
- return nil
|
|
|
- }
|
|
|
-
|
|
|
- u.gso.m.Lock()
|
|
|
- defer u.gso.m.Unlock()
|
|
|
-
|
|
|
- if !u.enableGSO || !p.Addr.IsValid() || len(p.Payload) > u.gso.MaxBytes {
|
|
|
- if err := u.flushGSOlocked(); err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
- return errGSOFallback
|
|
|
- }
|
|
|
-
|
|
|
- if len(u.gso.packets) == 0 {
|
|
|
- u.gso.setNameLocked(p.Addr, u.isV4)
|
|
|
- u.gso.SegSize = len(p.Payload)
|
|
|
- u.gso.packets = append(u.gso.packets, p)
|
|
|
- } else if p.Addr != u.gso.Addr || len(p.Payload) != u.gso.SegSize {
|
|
|
- if err := u.flushGSOlocked(); err != nil {
|
|
|
- return err
|
|
|
- } //todo deal with "one small packet" case
|
|
|
- u.gso.setNameLocked(p.Addr, u.isV4)
|
|
|
- u.gso.SegSize = len(p.Payload)
|
|
|
- u.gso.packets = append(u.gso.packets, p)
|
|
|
- } else {
|
|
|
- u.gso.packets = append(u.gso.packets, p)
|
|
|
- }
|
|
|
-
|
|
|
- //big todo
|
|
|
- //if len(u.gso.Buf)+len(p.Payload) > u.gso.MaxBytes {
|
|
|
- // if err := u.flushGSOlocked(); err != nil {
|
|
|
- // return err
|
|
|
- // }
|
|
|
- // u.gso.setNameLocked(p.Addr, u.isV4)
|
|
|
- // u.gso.SegSize = len(p.Payload)
|
|
|
- // u.gso.packets = append(u.gso.packets, p)
|
|
|
- //}
|
|
|
-
|
|
|
- if len(u.gso.packets) >= u.gso.MaxSegments || u.gso.FlushTimeout <= 0 {
|
|
|
- return u.flushGSOlocked()
|
|
|
- }
|
|
|
-
|
|
|
- u.scheduleGSOFlushLocked()
|
|
|
- return nil
|
|
|
-}
|
|
|
-
|
|
|
-func (u *StdConn) flushGSOlocked() error {
|
|
|
- if len(u.gso.packets) == 0 {
|
|
|
- u.stopGSOTimerLocked()
|
|
|
- return nil
|
|
|
- }
|
|
|
-
|
|
|
- u.stopGSOTimerLocked()
|
|
|
-
|
|
|
- if u.gso.SegSize <= 0 {
|
|
|
- return errGSOFallback
|
|
|
- }
|
|
|
-
|
|
|
- err := u.gso.sendmsgLocked(u.sysFd)
|
|
|
- if errors.Is(err, errGSODisabled) {
|
|
|
- u.l.WithField("addr", u.gso.Addr).Warn("UDP GSO disabled by kernel, falling back to sendto")
|
|
|
- u.enableGSO = false
|
|
|
- //todo!
|
|
|
- //return u.sendSegmentsIndividually(payload, addr, segSize)
|
|
|
- }
|
|
|
- u.gso.SegSize = 0
|
|
|
-
|
|
|
- return err
|
|
|
-}
|
|
|
-
|
|
|
-func (u *StdConn) gsoFlushTimer() {
|
|
|
- u.gso.m.Lock()
|
|
|
- _ = u.flushGSOlocked()
|
|
|
- u.gso.m.Unlock()
|
|
|
+ return u.writeTo6(b, ip)
|
|
|
}
|
|
|
|
|
|
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
|
|
|
@@ -578,7 +344,6 @@ func (u *StdConn) ReloadConfig(c *config.C) {
|
|
|
}
|
|
|
}
|
|
|
u.configureGRO(true)
|
|
|
- u.configureGSO(c)
|
|
|
}
|
|
|
|
|
|
func (u *StdConn) configureGRO(enable bool) {
|
|
|
@@ -602,49 +367,6 @@ func (u *StdConn) configureGRO(enable bool) {
|
|
|
u.enableGRO = false
|
|
|
}
|
|
|
|
|
|
-func (u *StdConn) configureGSO(c *config.C) {
|
|
|
- enable := c.GetBool("listen.enable_gso", true)
|
|
|
- if !enable {
|
|
|
- u.disableGSO()
|
|
|
- } else {
|
|
|
- u.enableGSO = true
|
|
|
- }
|
|
|
-
|
|
|
- segments := c.GetInt("listen.gso_max_segments", defaultGSOMaxSegments)
|
|
|
- if segments < 1 {
|
|
|
- segments = 1
|
|
|
- }
|
|
|
- u.gso.MaxSegments = segments
|
|
|
-
|
|
|
- maxBytes := c.GetInt("listen.gso_max_bytes", 0)
|
|
|
- if maxBytes <= 0 {
|
|
|
- maxBytes = MTU * segments
|
|
|
- }
|
|
|
- if maxBytes > maxGSOBatchBytes {
|
|
|
- u.l.WithField("requested", maxBytes).Warn("listen.gso_max_bytes larger than UDP limit; clamping")
|
|
|
- maxBytes = maxGSOBatchBytes
|
|
|
- }
|
|
|
- u.gso.MaxBytes = maxBytes
|
|
|
-
|
|
|
- timeout := c.GetDuration("listen.gso_flush_timeout", defaultGSOFlushTimeout)
|
|
|
- if timeout < 0 {
|
|
|
- timeout = 0
|
|
|
- }
|
|
|
- u.gso.FlushTimeout = timeout
|
|
|
- u.gso.Init()
|
|
|
-}
|
|
|
-
|
|
|
-func (u *StdConn) disableGSO() {
|
|
|
- u.gso.m.Lock()
|
|
|
- defer u.gso.m.Unlock()
|
|
|
- u.enableGSO = false
|
|
|
- _ = u.flushGSOlocked()
|
|
|
- u.gso.Buf = nil
|
|
|
- u.gso.packets = u.gso.packets[:0]
|
|
|
- u.gso.SegSize = 0
|
|
|
- u.stopGSOTimerLocked()
|
|
|
-}
|
|
|
-
|
|
|
func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
|
|
|
var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
|
|
|
_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
|
JackDoan