From 7a58331a2c3513e611741cd4ef56f188be37916d Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Fri, 5 Jul 2024 12:26:56 -0500 Subject: [PATCH] feat: convert execs to ip to netlink calls Not making direct exec calls to user binary interfaces has long been a principle of kube-router. When kube-router was first coded, the netlink library was missing significant features that forced us to exec out. However, now netlink seems to have most of the functionality that we need. This converts all of the places where we can use netlink to use the netlink functionality. --- pkg/controllers/proxy/linux_networking.go | 201 ++++++++++++------ .../proxy/network_services_controller.go | 43 ++-- pkg/controllers/proxy/utils.go | 9 - .../routing/network_routes_controller.go | 60 +++--- pkg/controllers/routing/pbr.go | 47 ++-- pkg/controllers/routing/utils.go | 50 ++--- pkg/utils/linux_routing.go | 4 +- 7 files changed, 246 insertions(+), 168 deletions(-) diff --git a/pkg/controllers/proxy/linux_networking.go b/pkg/controllers/proxy/linux_networking.go index 2fe37b3b3f..162d02c3bb 100644 --- a/pkg/controllers/proxy/linux_networking.go +++ b/pkg/controllers/proxy/linux_networking.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "os" - "os/exec" "path" "strconv" "strings" @@ -23,8 +22,10 @@ import ( ) const ( - ipv4NetMaskBits = 32 - ipv6NetMaskBits = 128 + ipv4NetMaskBits = 32 + ipv4DefaultRoute = "0.0.0.0/0" + ipv6NetMaskBits = 128 + ipv6DefaultRoute = "::/0" // TODO: it's bad to rely on eth0 here. While this is inside the container's namespace and is determined by the // container runtime and so far we've been able to count on this being reliably set to eth0, it is possible that @@ -66,7 +67,6 @@ type netlinkCalls interface { func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP string) error { var netMask net.IPMask - var ipRouteCmdArgs []string parsedIP := net.ParseIP(ip) parsedNodeIP := net.ParseIP(nodeIP) if parsedIP.To4() != nil { @@ -76,7 +76,6 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv4NetMaskBits, ipv4NetMaskBits) - ipRouteCmdArgs = make([]string, 0) } else { // If the IP family of the NodeIP and the VIP IP don't match, we can't proceed if parsedNodeIP.To4() != nil { @@ -89,7 +88,6 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv6NetMaskBits, ipv6NetMaskBits) - ipRouteCmdArgs = []string{"-6"} } naddr := &netlink.Addr{IPNet: &net.IPNet{IP: parsedIP, Mask: netMask}, Scope: syscall.RT_SCOPE_LINK} @@ -107,13 +105,20 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin // Delete VIP addition to "local" rt table also, fail silently if not found (DSR special case) // #nosec G204 - ipRouteCmdArgs = append(ipRouteCmdArgs, "route", "delete", "local", ip, "dev", KubeDummyIf, - "table", "local", "proto", "kernel", "scope", "host", "src", nodeIP, "table", "local") - out, err := exec.Command("ip", ipRouteCmdArgs...).CombinedOutput() + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: &net.IPNet{IP: parsedIP, Mask: netMask}, + LinkIndex: iface.Attrs().Index, + Table: syscall.RT_TABLE_LOCAL, + Protocol: unix.RTPROT_KERNEL, + Scope: syscall.RT_SCOPE_HOST, + Src: parsedNodeIP, + } + err = netlink.RouteDel(nRoute) if err != nil { - if !strings.Contains(string(out), "No such process") { - klog.Errorf("Failed to delete route to service VIP %s configured on %s. Error: %v, Output: %s", - ip, KubeDummyIf, err, out) + if !strings.Contains(err.Error(), "no such process") { + klog.Errorf("Failed to delete route to service VIP %s configured on %s. Error: %v", + ip, iface.Attrs().Name, err) } else { klog.Warningf("got a No such process error while trying to remove route: %v (this is not normally bad "+ "enough to stop processing)", err) @@ -129,7 +134,6 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin // inside the container. func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP string, addRoute bool) error { var netMask net.IPMask - var ipRouteCmdArgs []string var isIPv6 bool parsedIP := net.ParseIP(ip) parsedNodeIP := net.ParseIP(nodeIP) @@ -140,7 +144,6 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv4NetMaskBits, ipv4NetMaskBits) - ipRouteCmdArgs = make([]string, 0) isIPv6 = false } else { // If we're supposed to add a route and the IP family of the NodeIP and the VIP IP don't match, we can't proceed @@ -149,11 +152,11 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv6NetMaskBits, ipv6NetMaskBits) - ipRouteCmdArgs = []string{"-6"} isIPv6 = true } - naddr := &netlink.Addr{IPNet: &net.IPNet{IP: parsedIP, Mask: netMask}, Scope: syscall.RT_SCOPE_LINK} + ipPrefix := &net.IPNet{IP: parsedIP, Mask: netMask} + naddr := &netlink.Addr{IPNet: ipPrefix, Scope: syscall.RT_SCOPE_LINK} err := netlink.AddrAdd(iface, naddr) if err != nil && err.Error() != IfaceHasAddr { klog.Errorf("failed to assign cluster ip %s to dummy interface: %s", naddr.IPNet.IP.String(), err.Error()) @@ -168,16 +171,24 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP strin return nil } - // TODO: netlink.RouteReplace which is replacement for below command is not working as expected. Call succeeds but - // route is not replaced. For now do it with command. - // #nosec G204 - ipRouteCmdArgs = append(ipRouteCmdArgs, "route", "replace", "local", ip, "dev", KubeDummyIf, - "table", "local", "proto", "kernel", "scope", "host", "src", nodeIP, "table", "local") - - out, err := exec.Command("ip", ipRouteCmdArgs...).CombinedOutput() + kubeDummyLink, err := netlink.LinkByName(KubeDummyIf) if err != nil { - klog.Errorf("Failed to replace route to service VIP %s configured on %s. Error: %v, Output: %s", - ip, KubeDummyIf, err, out) + klog.Errorf("failed to get %s link due to %v", KubeDummyIf, err) + return err + } + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: ipPrefix, + LinkIndex: kubeDummyLink.Attrs().Index, + Table: syscall.RT_TABLE_LOCAL, + Protocol: unix.RTPROT_KERNEL, + Scope: syscall.RT_SCOPE_HOST, + Src: parsedNodeIP, + } + err = netlink.RouteReplace(nRoute) + if err != nil { + klog.Errorf("Failed to replace route to service VIP %s configured on %s. Error: %v", + ip, KubeDummyIf, err) return err } @@ -462,24 +473,56 @@ func (ln *linuxNetworking) setupPolicyRoutingForDSR(setupIPv4, setupIPv6 bool) e return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err) } + loNetLink, err := netlink.LinkByName("lo") + if err != nil { + return fmt.Errorf("failed to get loopback interface due to %v", err) + } + if setupIPv4 { - out, err := exec.Command("ip", "route", "list", "table", customDSRRouteTableID).Output() - if err != nil || !strings.Contains(string(out), " lo ") { - if err = exec.Command("ip", "route", "add", "local", "default", "dev", "lo", "table", - customDSRRouteTableID).Run(); err != nil { - return fmt.Errorf("failed to add route in custom route table due to: %v", err) + nFamily := netlink.FAMILY_V4 + _, defaultRouteCIDR, err := net.ParseCIDR(ipv4DefaultRoute) + if err != nil { + //nolint:goconst // This is a static value and should not be changed + return fmt.Errorf("failed to parse default (%s) route (this is statically defined, so if you see this "+ + "error please report because something has gone very wrong) due to: %v", ipv4DefaultRoute, err) + } + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: defaultRouteCIDR, + LinkIndex: loNetLink.Attrs().Index, + Table: customDSRRouteTableID, + } + routes, err := netlink.RouteListFiltered(nFamily, nRoute, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil || len(routes) < 1 { + err = netlink.RouteAdd(nRoute) + if err != nil { + return fmt.Errorf("failed to add route to custom route table for DSR due to: %v", err) } } } + if setupIPv6 { - out, err := exec.Command("ip", "-6", "route", "list", "table", customDSRRouteTableID).Output() - if err != nil || !strings.Contains(string(out), " lo ") { - if err = exec.Command("ip", "-6", "route", "add", "local", "default", "dev", "lo", "table", - customDSRRouteTableID).Run(); err != nil { - return fmt.Errorf("failed to add route in custom route table due to: %v", err) + nFamily := netlink.FAMILY_V6 + _, defaultRouteCIDR, err := net.ParseCIDR(ipv6DefaultRoute) + if err != nil { + return fmt.Errorf("failed to parse default (%s) route (this is statically defined, so if you see this "+ + "error please report because something has gone very wrong) due to: %v", ipv6DefaultRoute, err) + } + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: defaultRouteCIDR, + LinkIndex: loNetLink.Attrs().Index, + Table: customDSRRouteTableID, + } + routes, err := netlink.RouteListFiltered(nFamily, nRoute, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil || len(routes) < 1 { + err = netlink.RouteAdd(nRoute) + if err != nil { + return fmt.Errorf("failed to add route to custom route table for DSR due to: %v", err) } } } + return nil } @@ -487,7 +530,6 @@ func (ln *linuxNetworking) setupPolicyRoutingForDSR(setupIPv4, setupIPv6 bool) e // directly responds back with source IP as external IP kernel will treat as martian packet. // To prevent martian packets add route to external IP through the `kube-bridge` interface // setupRoutesForExternalIPForDSR: setups routing so that kernel does not think return packets as martians - func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap serviceInfoMap, setupIPv4, setupIPv6 bool) error { err := utils.RouteTableAdd(externalIPRouteTableID, externalIPRouteTableName) @@ -495,27 +537,45 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err) } - setupIPRulesAndRoutes := func(ipArgs []string) error { - out, err := runIPCommandsWithArgs(ipArgs, "rule", "list").Output() + setupIPRulesAndRoutes := func(isIPv6 bool) error { + nFamily := netlink.FAMILY_V4 + _, defaultPrefixCIDR, err := net.ParseCIDR(ipv4DefaultRoute) + if isIPv6 { + nFamily = netlink.FAMILY_V6 + _, defaultPrefixCIDR, err = net.ParseCIDR(ipv6DefaultRoute) + } + if err != nil { + return fmt.Errorf("failed to parse default route (this is statically defined, so if you see this "+ + "error please report because something has gone very wrong) due to: %v", err) + } + + nRule := &netlink.Rule{ + Priority: defaultDSRPolicyRulePriority, + Src: defaultPrefixCIDR, + Table: externalIPRouteTableID, + } + rules, err := netlink.RuleListFiltered(nFamily, nRule, + netlink.RT_FILTER_TABLE|netlink.RT_FILTER_SRC|netlink.RT_FILTER_PRIORITY) if err != nil { - return fmt.Errorf("failed to verify if `ip rule add prio 32765 from all lookup external_ip` exists due to: %v", - err) + return fmt.Errorf("failed to list rule for external IP's and verify if `ip rule add prio 32765 from all "+ + "lookup external_ip` exists due to: %v", err) } - if !(strings.Contains(string(out), externalIPRouteTableName) || - strings.Contains(string(out), externalIPRouteTableID)) { - err = runIPCommandsWithArgs(ipArgs, "rule", "add", "prio", "32765", "from", "all", "lookup", - externalIPRouteTableID).Run() + if len(rules) < 1 { + err = netlink.RuleAdd(nRule) if err != nil { klog.Infof("Failed to add policy rule `ip rule add prio 32765 from all lookup external_ip` due to %v", - err.Error()) + err) return fmt.Errorf("failed to add policy rule `ip rule add prio 32765 from all lookup external_ip` "+ "due to %v", err) } } - out, _ = runIPCommandsWithArgs(ipArgs, "route", "list", "table", externalIPRouteTableID).Output() - outStr := string(out) + kubeBridgeLink, err := netlink.LinkByName(KubeBridgeIf) + if err != nil { + return fmt.Errorf("failed to get kube-bridge interface due to %v", err) + } + activeExternalIPs := make(map[string]bool) for _, svc := range serviceInfoMap { for _, externalIP := range svc.externalIPs { @@ -528,9 +588,21 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service activeExternalIPs[externalIP] = true - if !strings.Contains(outStr, externalIP) { - if err = runIPCommandsWithArgs(ipArgs, "route", "add", externalIP, "dev", "kube-bridge", "table", - externalIPRouteTableID).Run(); err != nil { + nSrcIP := net.ParseIP(externalIP) + nRoute := &netlink.Route{ + Src: nSrcIP, + LinkIndex: kubeBridgeLink.Attrs().Index, + Table: externalIPRouteTableID, + } + + routes, err := netlink.RouteListFiltered(nFamily, nRoute, + netlink.RT_FILTER_SRC|netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil { + return fmt.Errorf("failed to list route for external IP's due to: %s", err) + } + if len(routes) < 1 { + err = netlink.RouteAdd(nRoute) + if err != nil { klog.Errorf("Failed to add route for %s in custom route table for external IP's due to: %v", externalIP, err) continue @@ -540,19 +612,18 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service } // check if there are any pbr in externalIPRouteTableID for external IP's - if len(outStr) > 0 { - // clean up stale external IPs - for _, line := range strings.Split(strings.Trim(outStr, "\n"), "\n") { - route := strings.Split(strings.Trim(line, " "), " ") - ip := route[0] - if !activeExternalIPs[ip] { - args := []string{"route", "del", "table", externalIPRouteTableID} - args = append(args, route...) - if err = runIPCommandsWithArgs(ipArgs, args...).Run(); err != nil { - klog.Errorf("Failed to del route for %v in custom route table for external IP's due to: %s", - ip, err) - continue - } + routes, err := netlink.RouteList(nil, nFamily) + if err != nil { + return fmt.Errorf("failed to list route for external IP's due to: %s", err) + } + for idx, route := range routes { + ip := route.Src.String() + if !activeExternalIPs[ip] { + err = netlink.RouteDel(&routes[idx]) + if err != nil { + klog.Errorf("Failed to del route for %v in custom route table for external IP's due to: %s", + ip, err) + continue } } } @@ -561,13 +632,13 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service } if setupIPv4 { - err = setupIPRulesAndRoutes([]string{}) + err = setupIPRulesAndRoutes(false) if err != nil { return err } } if setupIPv6 { - err = setupIPRulesAndRoutes([]string{"-6"}) + err = setupIPRulesAndRoutes(true) if err != nil { return err } diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index cbe57ffcd0..67009ab59f 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -32,6 +32,7 @@ const ( KubeDummyIf = "kube-dummy-if" KubeTunnelIfv4 = "kube-tunnel-if" KubeTunnelIfv6 = "kube-tunnel-v6" + KubeBridgeIf = "kube-bridge" IfaceNotFound = "Link not found" IfaceHasAddr = "file exists" IfaceHasNoAddr = "cannot assign requested address" @@ -41,11 +42,13 @@ const ( IpvsSvcFSched2 = "flag-2" IpvsSvcFSched3 = "flag-3" - customDSRRouteTableID = "78" - customDSRRouteTableName = "kube-router-dsr" - externalIPRouteTableID = "79" - externalIPRouteTableName = "external_ip" - kubeRouterProxyName = "kube-router" + customDSRRouteTableID = 78 + customDSRRouteTableName = "kube-router-dsr" + externalIPRouteTableID = 79 + externalIPRouteTableName = "external_ip" + kubeRouterProxyName = "kube-router" + defaultTrafficDirectorRulePriority = 32764 + defaultDSRPolicyRulePriority = 32765 // Taken from https://github.com/torvalds/linux/blob/master/include/uapi/linux/ip_vs.h#L21 ipvsPersistentFlagHex = 0x0001 @@ -1721,23 +1724,35 @@ func (nsc *NetworkServicesController) cleanupMangleTableRule(ip string, protocol // http://www.austintek.com/LVS/LVS-HOWTO/HOWTO/LVS-HOWTO.routing_to_VIP-less_director.html // routeVIPTrafficToDirector: setups policy routing so that FWMARKed packets are delivered locally func routeVIPTrafficToDirector(fwmark string, family v1.IPFamily) error { - ipArgs := make([]string, 0) + nFamily := netlink.FAMILY_V4 if family == v1.IPv6Protocol { - ipArgs = append(ipArgs, "-6") + nFamily = netlink.FAMILY_V6 } - out, err := runIPCommandsWithArgs(ipArgs, "rule", "list").Output() + iFWMark, err := strconv.Atoi(fwmark) if err != nil { - return errors.New("Failed to verify if `ip rule` exists due to: " + err.Error()) + return fmt.Errorf("failed to convert fwmark to integer due to: %v", err) } - if !strings.Contains(string(out), fwmark+" ") { - err = runIPCommandsWithArgs(ipArgs, "rule", "add", "prio", "32764", "fwmark", fwmark, "table", - customDSRRouteTableID).Run() + + nRule := &netlink.Rule{ + Mark: iFWMark, + Table: customDSRRouteTableID, + Priority: defaultTrafficDirectorRulePriority, + } + + routes, err := netlink.RuleListFiltered(nFamily, nRule, netlink.RT_FILTER_MARK|netlink.RT_FILTER_TABLE) + if err != nil { + return fmt.Errorf("failed to verify if `ip rule` exists due to: %v", err) + } + + if len(routes) < 1 { + err = netlink.RuleAdd(nRule) if err != nil { - return errors.New("Failed to add policy rule to lookup traffic to VIP through the custom " + - " routing table due to " + err.Error()) + return fmt.Errorf("failed to add policy rule to lookup traffic to VIP through the custom "+ + "routing table due to %v", err) } } + return nil } diff --git a/pkg/controllers/proxy/utils.go b/pkg/controllers/proxy/utils.go index 5afe608985..0b61b4b9a2 100644 --- a/pkg/controllers/proxy/utils.go +++ b/pkg/controllers/proxy/utils.go @@ -6,7 +6,6 @@ import ( "fmt" "hash/fnv" "net" - "os/exec" "runtime" "strconv" "strings" @@ -559,14 +558,6 @@ func getIPVSFirewallInputChainRule(family v1.IPFamily) []string { "-j", ipvsFirewallChainName} } -// runIPCommandsWithArgs extend the exec.Command interface to allow passing an additional array of arguments to ip -func runIPCommandsWithArgs(ipArgs []string, additionalArgs ...string) *exec.Cmd { - var allArgs []string - allArgs = append(allArgs, ipArgs...) - allArgs = append(allArgs, additionalArgs...) - return exec.Command("ip", allArgs...) -} - // getLabelFromMap checks the list of passed labels for the service.kubernetes.io/service-proxy-name // label and if it exists, returns it otherwise returns an error func getLabelFromMap(label string, labels map[string]string) (string, error) { diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index d6450d994e..688ef1e12c 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -33,7 +33,7 @@ import ( const ( IfaceNotFound = "Link not found" - customRouteTableID = "77" + customRouteTableID = 77 customRouteTableName = "kube-router" podSubnetsIPSetName = "kube-router-pod-subnets" nodeAddrsIPSetName = "kube-router-node-ips" @@ -753,11 +753,11 @@ func (nrc *NetworkRoutingController) cleanupTunnel(destinationSubnet *net.IPNet, // setupOverlayTunnel attempts to create a tunnel link and corresponding routes for IPIP based overlay networks func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextHop net.IP, nextHopSubnet *net.IPNet) (netlink.Link, error) { - var out []byte link, err := netlink.LinkByName(tunnelName) var bestIPForFamily net.IP var ipipMode, fouLinkType string + var nFamily int isIPv6 := false ipBase := make([]string, 0) strFormattedEncapPort := strconv.FormatInt(int64(nrc.overlayEncapPort), 10) @@ -766,12 +766,14 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH bestIPForFamily = utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs) ipipMode = encapTypeIPIP fouLinkType = ipipModev4 + nFamily = netlink.FAMILY_V4 } else { // Need to activate the ip command in IPv6 mode ipBase = append(ipBase, "-6") bestIPForFamily = utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs) ipipMode = ipipModev6 fouLinkType = "ip6tnl" + nFamily = netlink.FAMILY_V6 isIPv6 = true } if nil == bestIPForFamily { @@ -799,13 +801,17 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH // If we are transitioning from FoU to IPIP we also need to clean up the old FoU port if it exists if fouPortAndProtoExist(nrc.overlayEncapPort, isIPv6) { - fouArgs := ipBase - fouArgs = append(fouArgs, "fou", "del", "port", strFormattedEncapPort) - out, err := exec.Command("ip", fouArgs...).CombinedOutput() + nFOU := &netlink.Fou{ + Family: nFamily, + Port: int(nrc.overlayEncapPort), + EncapType: netlink.FOU_ENCAP_GUE, + } + + err = netlink.FouDel(*nFOU) if err != nil { - klog.Warningf("failed to clean up previous FoU tunnel port (this is only a warning because it "+ - "won't stop kube-router from working for now, but still shouldn't have happened) - error: "+ - "%v, output %s", err, out) + klog.Errorf("failed to clean up previous FoU tunnel (%s) port (this is only a warning because "+ + "it won't stop kube-router from working for now, but still shouldn't have happened) - "+ + "error: %v", tunnelName, err) } } } @@ -835,13 +841,16 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH case encapTypeFOU: // Ensure that the FOU tunnel port is set correctly if !fouPortAndProtoExist(nrc.overlayEncapPort, isIPv6) { - fouArgs := ipBase - fouArgs = append(fouArgs, "fou", "add", "port", strFormattedEncapPort, "gue") - out, err := exec.Command("ip", fouArgs...).CombinedOutput() + nFOU := netlink.Fou{ + Family: nFamily, + Port: int(nrc.overlayEncapPort), + EncapType: netlink.FOU_ENCAP_GUE, + } + err = netlink.FouAdd(nFOU) if err != nil { - //nolint:goconst // don't need to make error messages a constant + //nolint:goconst // This does not need to be abstracted return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+ - "Failed to set FoU tunnel port - error: %s, output: %s", tunnelName, err, string(out)) + "Failed to set FoU tunnel port - error: %s", tunnelName, err) } } @@ -856,6 +865,10 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH } klog.V(2).Infof("Executing the following command to create tunnel: ip %s", cmdArgs) + // TODO: we should make this call via the netlink library whenever they support ip6ip6 tunnels. Right now, it + // doesn't appear like the project supports them. They support something else called an ip6tun which seems + // similar, but we would probably need to figure out what combination of primitives we need to serialize in + // order to get the same effect we get below. out, err := exec.Command("ip", cmdArgs...).CombinedOutput() if err != nil { return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+ @@ -875,17 +888,16 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH // Now that the tunnel link exists, we need to add a route to it, so the node knows where to send traffic bound for // this interface - //nolint:gocritic // we understand that we are appending to a new slice - cmdArgs := append(ipBase, "route", "list", "table", customRouteTableID) - out, err = exec.Command("ip", cmdArgs...).CombinedOutput() - // This used to be "dev "+tunnelName+" scope" but this isn't consistent with IPv6's output, so we changed it to just - // "dev "+tunnelName, but at this point I'm unsure if there was a good reason for adding scope on before, so that's - // why this comment is here. - if err != nil || !strings.Contains(string(out), "dev "+tunnelName) { - //nolint:gocritic // we understand that we are appending to a new slice - cmdArgs = append(ipBase, "route", "add", nextHop.String(), "dev", tunnelName, "table", customRouteTableID) - if out, err = exec.Command("ip", cmdArgs...).CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to add route in custom route table, err: %s, output: %s", err, string(out)) + nRoute := &netlink.Route{ + Gw: nextHop, + LinkIndex: link.Attrs().Index, + Table: customRouteTableID, + } + foundRoutes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, nRoute, + netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil || len(foundRoutes) < 1 { + if err = netlink.RouteAdd(nRoute); err != nil { + return nil, fmt.Errorf("failed to list routes in custom route table %d, error: %v", customRouteTableID, err) } } diff --git a/pkg/controllers/routing/pbr.go b/pkg/controllers/routing/pbr.go index cd93a27edb..a9c34995f7 100644 --- a/pkg/controllers/routing/pbr.go +++ b/pkg/controllers/routing/pbr.go @@ -2,31 +2,44 @@ package routing import ( "fmt" - "os/exec" - "strings" + "net" "github.com/cloudnativelabs/kube-router/v2/pkg/utils" + "github.com/vishvananda/netlink" +) + +const ( + PBRRuleAdd = iota + PBRRuleDel ) // ipRuleAbstraction used for abstracting iproute2 rule additions between IPv4 and IPv6 for both add and del operations. // ipProtocol is the iproute2 protocol specified as a string ("-4" or "-6"). ipOp is the rule operation specified as a // string ("add" or "del). The cidr is the IPv4 / IPv6 source CIDR string that when received will be used to lookup // routes in a custom table. -func ipRuleAbstraction(ipProtocol, ipOp, cidr string) error { - out, err := exec.Command("ip", ipProtocol, "rule", "list").Output() +func ipRuleAbstraction(ipFamily int, ipOp int, cidr string) error { + _, nSrc, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("failed to parse CIDR: %s", err.Error()) + } + + nRule := &netlink.Rule{ + Family: ipFamily, + Src: nSrc, + Table: customRouteTableID, + } + rules, err := netlink.RuleListFiltered(ipFamily, nRule, netlink.RT_FILTER_SRC) if err != nil { - return fmt.Errorf("failed to verify if `ip rule` exists: %s", err.Error()) + return fmt.Errorf("failed to list rules: %s", err.Error()) } - if strings.Contains(string(out), cidr) && ipOp == "del" { - err = exec.Command("ip", ipProtocol, "rule", ipOp, "from", cidr, "lookup", customRouteTableID).Run() - if err != nil { - return fmt.Errorf("failed to add ip rule due to: %s", err.Error()) + if ipOp == PBRRuleDel && len(rules) > 0 { + if err := netlink.RuleDel(nRule); err != nil { + return fmt.Errorf("failed to delete rule: %s", err.Error()) } - } else if !strings.Contains(string(out), cidr) && ipOp == "add" { - err = exec.Command("ip", ipProtocol, "rule", ipOp, "from", cidr, "lookup", customRouteTableID).Run() - if err != nil { - return fmt.Errorf("failed to add ip rule due to: %s", err.Error()) + } else if ipOp == PBRRuleAdd && len(rules) < 1 { + if err := netlink.RuleAdd(nRule); err != nil { + return fmt.Errorf("failed to add rule: %s", err.Error()) } } @@ -43,14 +56,14 @@ func (nrc *NetworkRoutingController) enablePolicyBasedRouting() error { if nrc.isIPv4Capable { for _, ipv4CIDR := range nrc.podIPv4CIDRs { - if err := ipRuleAbstraction("-4", "add", ipv4CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V4, PBRRuleAdd, ipv4CIDR); err != nil { return err } } } if nrc.isIPv6Capable { for _, ipv6CIDR := range nrc.podIPv6CIDRs { - if err := ipRuleAbstraction("-6", "add", ipv6CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V6, PBRRuleAdd, ipv6CIDR); err != nil { return err } } @@ -67,14 +80,14 @@ func (nrc *NetworkRoutingController) disablePolicyBasedRouting() error { if nrc.isIPv4Capable { for _, ipv4CIDR := range nrc.podIPv4CIDRs { - if err := ipRuleAbstraction("-4", "del", ipv4CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V4, PBRRuleDel, ipv4CIDR); err != nil { return err } } } if nrc.isIPv6Capable { for _, ipv6CIDR := range nrc.podIPv6CIDRs { - if err := ipRuleAbstraction("-6", "del", ipv6CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V6, PBRRuleDel, ipv6CIDR); err != nil { return err } } diff --git a/pkg/controllers/routing/utils.go b/pkg/controllers/routing/utils.go index 455240e9fa..d9a330dc84 100644 --- a/pkg/controllers/routing/utils.go +++ b/pkg/controllers/routing/utils.go @@ -1,13 +1,11 @@ package routing import ( - "bufio" "crypto/sha256" "encoding/base64" "errors" "fmt" "net" - "os/exec" "regexp" "strconv" "strings" @@ -336,41 +334,20 @@ func (nrc *NetworkRoutingController) getBGPRouteInfoForVIP(vip string) (subnet u // where the only thing that distinguishes them is the -6 or not on the end // WARNING we're parsing a CLI tool here not an API, this may break at some point in the future func fouPortAndProtoExist(port uint16, isIPv6 bool) bool { - const ipRoute2IPv6Prefix = "-6" - strPort := strconv.FormatInt(int64(port), 10) - fouArgs := make([]string, 0) - klog.V(2).Infof("Checking FOU Port and Proto... %s - %t", strPort, isIPv6) + klog.V(2).Infof("Checking FOU Port and Proto... %d - %t", port, isIPv6) + fouFamily := netlink.FAMILY_V4 if isIPv6 { - fouArgs = append(fouArgs, ipRoute2IPv6Prefix) + fouFamily = netlink.FAMILY_V6 } - fouArgs = append(fouArgs, "fou", "show") - - out, err := exec.Command("ip", fouArgs...).CombinedOutput() - // iproute2 returns an error if no fou configuration exists + fous, err := netlink.FouList(fouFamily) if err != nil { + klog.Errorf("failed to list fou ports: %v", err) return false } - strOut := string(out) - klog.V(2).Infof("Combined output of ip fou show: %s", strOut) - scanner := bufio.NewScanner(strings.NewReader(strOut)) - - // loop over all lines of output - for scanner.Scan() { - scannedLine := scanner.Text() - // if the output doesn't contain our port at all, then continue - if !strings.Contains(scannedLine, strPort) { - continue - } - - // if this is IPv6 port and it has the correct IPv6 suffix (see example above) then return true - if isIPv6 && strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) { - return true - } - - // if this is not IPv6 and it does not have an IPv6 suffix (see example above) then return true - if !isIPv6 && !strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) { + for _, fou := range fous { + if fou.Port == int(port) && fou.EncapType == netlink.FOU_ENCAP_GUE { return true } } @@ -387,18 +364,17 @@ func fouPortAndProtoExist(port uint16, isIPv6 bool) bool { // Output for a normal IPIP tunnel looks like: // ipip ipip remote local dev ttl inherit ... func linkFOUEnabled(linkName string) bool { - const fouEncapEnabled = "encap gue" - cmdArgs := []string{"-details", "link", "show", linkName} - - out, err := exec.Command("ip", cmdArgs...).CombinedOutput() + const fouEncapType = "gue" + nLink, err := netlink.LinkByName(linkName) if err != nil { - klog.Warningf("recevied an error while trying to look at the link details of %s, this shouldn't have happened", - linkName) + klog.Errorf("recevied an error while trying to look at the link details of %s, this shouldn't have happened: "+ + "%v", linkName, err) return false + } - if strings.Contains(string(out), fouEncapEnabled) { + if nLink.Attrs().EncapType == fouEncapType { return true } diff --git a/pkg/utils/linux_routing.go b/pkg/utils/linux_routing.go index 78f2f3b527..0256a7880b 100644 --- a/pkg/utils/linux_routing.go +++ b/pkg/utils/linux_routing.go @@ -22,7 +22,7 @@ var ( ) // RouteTableAdd adds a new named table to iproute's rt_tables configuration file -func RouteTableAdd(tableNumber, tableName string) error { +func RouteTableAdd(tableNumber int, tableName string) error { var rtTablesLoc string for _, possibleLoc := range rtTablesPosLoc { _, err := os.Stat(possibleLoc) @@ -47,7 +47,7 @@ func RouteTableAdd(tableNumber, tableName string) error { return fmt.Errorf("failed to open: %s", err.Error()) } defer CloseCloserDisregardError(f) - if _, err = f.WriteString(tableNumber + " " + tableName + "\n"); err != nil { + if _, err = f.WriteString(fmt.Sprint(tableNumber) + " " + tableName + "\n"); err != nil { return fmt.Errorf("failed to write: %s", err.Error()) } }