diff --git a/third_party/meshnet/daemon/main.go b/third_party/meshnet/daemon/main.go index 7ea4e7ac..8ee6dfcd 100644 --- a/third_party/meshnet/daemon/main.go +++ b/third_party/meshnet/daemon/main.go @@ -22,6 +22,8 @@ func main() { } defer cni.Cleanup() + wireutil.TuneSystem() + isDebug := flag.Bool("d", false, "enable degugging") grpcPort, err := strconv.Atoi(os.Getenv("GRPC_PORT")) if err != nil || grpcPort == 0 { diff --git a/third_party/meshnet/daemon/vxlan/vxlan.go b/third_party/meshnet/daemon/vxlan/vxlan.go index 496cab76..9bf49c51 100644 --- a/third_party/meshnet/daemon/vxlan/vxlan.go +++ b/third_party/meshnet/daemon/vxlan/vxlan.go @@ -13,6 +13,7 @@ import ( "github.com/vishvananda/netlink" mpb "github.com/openconfig/kne/third_party/meshnet/daemon/proto/meshnet/v1beta1" + "github.com/openconfig/kne/third_party/meshnet/utils/wireutil" ) var vxLanOvrlyLogger *log.Entry = nil @@ -107,6 +108,20 @@ func CreateOrUpdate(v *mpb.RemotePod) error { } } + // Tune txqueuelen inside the container netns (configurable via LINK_TXQUEUELEN) + if podNs, err := ns.GetNS(veth.NsName); err == nil { + _ = podNs.Do(func(_ ns.NetNS) error { + if link, err := netlink.LinkByName(veth.LinkName); err == nil { + txqLen := wireutil.GetLinkTxQLen() + if err := netlink.LinkSetTxQLen(link, txqLen); err != nil { + vxLanOvrlyLogger.Warnf("failed to set txqueuelen %d on %s inside %s: %v", txqLen, veth.LinkName, veth.NsName, err) + } + } + return nil + }) + podNs.Close() + } + return nil } diff --git a/third_party/meshnet/utils/wireutil/sys_tune.go b/third_party/meshnet/utils/wireutil/sys_tune.go index 67291abb..782094aa 100644 --- a/third_party/meshnet/utils/wireutil/sys_tune.go +++ b/third_party/meshnet/utils/wireutil/sys_tune.go @@ -3,6 +3,9 @@ package wireutil import ( "os" "strconv" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // GetEnvInt reads an integer environment variable with a default fallback value if unset or invalid. @@ -14,3 +17,62 @@ func GetEnvInt(key string, defaultVal int) int { } return defaultVal } + +func getEnvString(key string, defaultVal string) string { + if valStr := os.Getenv(key); valStr != "" { + return valStr + } + return defaultVal +} + +// GetLinkTxQLen returns the configured link txqueuelen (default 10000, configurable via LINK_TXQUEUELEN). +func GetLinkTxQLen() int { + return GetEnvInt("LINK_TXQUEUELEN", 10000) +} + +// TuneSystem configures global OS sysctl tunables (backlog, buffers, ARP/neighbor GC thresholds, +// multicast group limits, rp_filter, and IPv6 startup behavior) and RLIMIT_NOFILE for high-density, +// high-throughput network topologies. Values can be customized via environment variables. +func TuneSystem() { + // 1. Increase max open file descriptors for daemon (rlimit) + noFileLimit := GetEnvInt("RLIMIT_NOFILE", 1048576) + var rlim unix.Rlimit + rlim.Max = uint64(noFileLimit) + rlim.Cur = uint64(noFileLimit) + if err := unix.Setrlimit(unix.RLIMIT_NOFILE, &rlim); err != nil { + log.Warnf("TuneSystem: failed to set RLIMIT_NOFILE to %d: %v", noFileLimit, err) + } else { + log.Infof("TuneSystem: successfully set RLIMIT_NOFILE to %d", noFileLimit) + } + + // 2. Sysctl kernel tunables for network device backlog, buffer limits, ARP/neighbor GC thresholds, + // multicast memberships, reverse path filtering, and IPv6 DAD/RS startup tuning. + sysctls := map[string]string{ + "/proc/sys/net/core/netdev_max_backlog": getEnvString("NETDEV_MAX_BACKLOG", "10000"), + "/proc/sys/net/core/rmem_max": getEnvString("RMEM_MAX", "16777216"), + "/proc/sys/net/core/wmem_max": getEnvString("WMEM_MAX", "16777216"), + "/proc/sys/net/core/rmem_default": getEnvString("RMEM_DEFAULT", "16777216"), + "/proc/sys/net/core/wmem_default": getEnvString("WMEM_DEFAULT", "16777216"), + "/proc/sys/net/ipv4/neigh/default/gc_thresh1": getEnvString("ARP_GC_THRESH1", "1024"), + "/proc/sys/net/ipv4/neigh/default/gc_thresh2": getEnvString("ARP_GC_THRESH2", "4096"), + "/proc/sys/net/ipv4/neigh/default/gc_thresh3": getEnvString("ARP_GC_THRESH3", "8192"), + "/proc/sys/net/ipv6/neigh/default/gc_thresh1": getEnvString("ARP_GC_THRESH1", "1024"), + "/proc/sys/net/ipv6/neigh/default/gc_thresh2": getEnvString("ARP_GC_THRESH2", "4096"), + "/proc/sys/net/ipv6/neigh/default/gc_thresh3": getEnvString("ARP_GC_THRESH3", "8192"), + "/proc/sys/net/ipv4/igmp_max_memberships": getEnvString("IGMP_MAX_MEMBERSHIPS", "10000"), + "/proc/sys/net/ipv6/mld_max_msf": getEnvString("MLD_MAX_MSF", "4096"), + "/proc/sys/net/ipv4/conf/all/rp_filter": getEnvString("RP_FILTER", "2"), + "/proc/sys/net/ipv4/conf/default/rp_filter": getEnvString("RP_FILTER", "2"), + "/proc/sys/net/ipv6/conf/default/accept_dad": getEnvString("IPV6_ACCEPT_DAD", "0"), + "/proc/sys/net/ipv6/conf/default/router_solicitations": getEnvString("IPV6_ROUTER_SOLICITATIONS", "0"), + "/proc/sys/net/ipv6/route/max_size": getEnvString("IPV6_ROUTE_MAX_SIZE", "1048576"), + } + + for path, val := range sysctls { + if err := os.WriteFile(path, []byte(val), 0644); err != nil { + log.Warnf("TuneSystem: failed to write %s to %s: %v", val, path, err) + } else { + log.Infof("TuneSystem: set %s = %s", path, val) + } + } +} diff --git a/third_party/meshnet/utils/wireutil/tap.go b/third_party/meshnet/utils/wireutil/tap.go index b0bdde76..21be3781 100644 --- a/third_party/meshnet/utils/wireutil/tap.go +++ b/third_party/meshnet/utils/wireutil/tap.go @@ -62,6 +62,12 @@ func CreateOrAttachTAP(podNsPath string, ifName string, ipCIDR string) (*os.File return fmt.Errorf("failed to find link %s inside netns %s: %w", ifName, podNsPath, err) } + // Increase txqueuelen for high-throughput packet processing (configurable via LINK_TXQUEUELEN) + txqLen := GetLinkTxQLen() + if err := netlink.LinkSetTxQLen(link, txqLen); err != nil { + log.Warnf("CreateOrAttachTAP: failed to set txqueuelen %d on %s in netns %s: %v", txqLen, ifName, podNsPath, err) + } + if err := netlink.LinkSetUp(link); err != nil { unix.Close(fd) return fmt.Errorf("failed to set %s UP in netns %s: %w", ifName, podNsPath, err) diff --git a/third_party/meshnet/utils/wireutil/veth.go b/third_party/meshnet/utils/wireutil/veth.go index 8add33e5..58ebfdb9 100644 --- a/third_party/meshnet/utils/wireutil/veth.go +++ b/third_party/meshnet/utils/wireutil/veth.go @@ -173,6 +173,12 @@ func ConfigurePodLinks(podNsPath string, links []PodLinkConfig) error { } } + // Increase txqueuelen for high-throughput packet processing (configurable via LINK_TXQUEUELEN) + txqLen := GetLinkTxQLen() + if err := netlink.LinkSetTxQLen(link, txqLen); err != nil { + log.Warnf("ConfigurePodLinks: failed to set txqueuelen %d on %s inside %s: %v", txqLen, cfg.LocalIntf, podNsPath, err) + } + if err := netlink.LinkSetUp(link); err != nil { return fmt.Errorf("failed to set %s UP inside %s: %w", cfg.LocalIntf, podNsPath, err) }