Skip to content

Commit

Permalink
nettools: Better support for old cni plugins
Browse files Browse the repository at this point in the history 
This change should improve overal support for cni plugins returning
[Result](https://github.com/containernetworking/cni/blob/master/SPEC.md#result)
in form compatible with v0.2.0 of specification.

E.x. of such plugin is `bridge` used in our e2e, but also [SR-IOV](hustcat/sriov-cni#22).
  • Loading branch information
@jellonek
jellonek committed Nov 16, 2017
1 parent 711ed1a commit 7ca71a3
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 101 deletions.
168 changes: 113 additions & 55 deletions pkg/nettools/nettools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,37 +318,114 @@ func FindVeth(links []netlink.Link) (netlink.Link, error) {
return veth, nil
}

func findLink(links []netlink.Link, iface *cnicurrent.Interface) (netlink.Link, error) {
func findLinkByAddress(links []netlink.Link, address net.IPNet) (netlink.Link, error) {
for _, link := range links {
if link.Attrs().Name == iface.Name {
return link, nil
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
return nil, err
}
for _, addr := range addresses {
if addr.String() == address.String() {
return link, nil
}
}
}
return nil, fmt.Errorf("interface with name %q not found in container namespace", iface.Name)
return nil, fmt.Errorf("interface with address %q not found in container namespace", address.String())
}

// GetContainerInterfaces locates veth network links in the current network namespace.
// If info is not nil and contains a non-empty list of network interfaces,
// the resulting slice will contain a link for each entry in that list.
// If info is nil or has no interfaces listed, the current network namespace
// must contain exactly one veth network link.
func GetContainerInterfaces(info *cnicurrent.Result) ([]netlink.Link, error) {
// ValidateAndfixCNIResult verifies that netConfig contains proper list of
// ips, routes, interfaces and if something is missing it tries to complement
// that using patch for Weave or for plugins which return their netConfig
// in v0.2.0 version of CNI SPEC
func ValidateAndfixCNIResult(netConfig *cnicurrent.Result) error {
allLinks, err := netlink.LinkList()
if err != nil {
return nil, fmt.Errorf("error listing links: %v", err)
return fmt.Errorf("error listing links: %v", err)
}

if info == nil || len(info.Interfaces) == 0 {
link, err := FindVeth(allLinks)
// If there are no routes provided, we consider it a broken
// config and extract interface config instead. That's the
// case with Weave CNI plugin.
if cni.GetPodIP(netConfig) == "" || len(netConfig.Routes) == 0 {
dnsInfo := netConfig.DNS

veth, err := FindVeth(allLinks)
if err != nil {
return nil, err
return err
}
if netConfig, err = ExtractLinkInfo(veth); err != nil {
return err
}
return []netlink.Link{link}, nil

// extracted netConfig doesn't have DNS information, so
// still try to extract it from CNI-provided data
netConfig.DNS = dnsInfo

return nil
}

if len(netConfig.IPs) == 0 {
return fmt.Errorf("cni result does not have any IP addresses")
}

// If on list of interfaces are missing elements matching these mentioned
// by interface index in elements of ip list and for all elements
// of this list which have value -1 for interface index - add them to list
// of interfaces and fix its index in ip list entry
if len(netConfig.Interfaces) == 0 {
alreadyDefindeLinks, err := GetContainerLinks(netConfig.Interfaces)
if err != nil {
return err
}

for _, ipConfig := range netConfig.IPs {
link, err := findLinkByAddress(allLinks, ipConfig.Address)
if err != nil {
return err
}

found := false
for i, l := range alreadyDefindeLinks {
if l == link {
ipConfig.Interface = i
found = true
break
}
}
if !found {
netConfig.Interfaces = append(netConfig.Interfaces, &cnicurrent.Interface{
Name: link.Attrs().Name,
Mac: link.Attrs().HardwareAddr.String(),
})
ipConfig.Interface = len(alreadyDefindeLinks)
alreadyDefindeLinks = append(alreadyDefindeLinks, link)
}
}
}

return nil
}

func findLinkByName(links []netlink.Link, name string) (netlink.Link, error) {
for _, link := range links {
if link.Attrs().Name == name {
return link, nil
}
}
return nil, fmt.Errorf("interface with name %q not found in container namespace", name)
}

// GetContainerLinks locates in container namespac enetwork links
// for provided interfaces
func GetContainerLinks(interfaces []*cnicurrent.Interface) ([]netlink.Link, error) {
allLinks, err := netlink.LinkList()
if err != nil {
return nil, fmt.Errorf("error listing links: %v", err)
}

var links []netlink.Link
for _, iface := range info.Interfaces {
link, err := findLink(allLinks, iface)
for _, iface := range interfaces {
link, err := findLinkByName(allLinks, iface.Name)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -517,53 +594,34 @@ type ContainerSideNetwork struct {
}

// SetupContainerSideNetwork sets up networking in container
// namespace. It does so by calling ExtractLinkInfo() first unless
// non-nil info argument is provided and then preparing the following
// namespace. It does so by preparing the following
// network interfaces in container ns:
// tap0 - tap interface for the VM
// br0 - a bridge that joins tap0 and original CNI veth
// The bridge (br0) gets assigned a link-local address to be used
// tapX - tap interface for the each interface to pass to VM
// brX - a bridge that joins above tapX and original CNI interface
// with X denoting an link index in info.Interfaces list.
// Each bridge gets assigned a link-local address to be used
// for dhcp server.
// The function should be called from within container namespace.
// Returns container network struct and an error, if any
func SetupContainerSideNetwork(info *cnicurrent.Result) (*ContainerSideNetwork, error) {
contVeths, err := GetContainerInterfaces(info)
contLinks, err := GetContainerLinks(info.Interfaces)
if err != nil {
return nil, err
}
// If there are no routes provided, we consider it a broken
// config and extract interface config instead. That's the
// case with Weave CNI plugin.
if info == nil || cni.GetPodIP(info) == "" || len(info.Routes) == 0 {
var dnsInfo cnitypes.DNS
if info != nil {
dnsInfo = info.DNS
}
if len(contVeths) == 0 {
return nil, errors.New("can not find network interfaces in container namespace")
}
info, err = ExtractLinkInfo(contVeths[0])
if err != nil {
return nil, err
}
// extracted info doesn't have DNS information, so
// still try to extract it from CNI-provided data
info.DNS = dnsInfo
}

var (
tapFiles []*os.File
hwAddrs []net.HardwareAddr
)

for i, contVeth := range contVeths {
hwAddr := contVeth.Attrs().HardwareAddr
for i, link := range contLinks {
hwAddr := link.Attrs().HardwareAddr
newHwAddr, err := GenerateMacAddress()
if err == nil {
err = SetHardwareAddr(contVeth, newHwAddr)
err = SetHardwareAddr(link, newHwAddr)
}
if err == nil {
err = StripLink(contVeth)
err = StripLink(link)
}
if err != nil {
return nil, err
Expand All @@ -574,7 +632,7 @@ func SetupContainerSideNetwork(info *cnicurrent.Result) (*ContainerSideNetwork,
LinkAttrs: netlink.LinkAttrs{
Name: tapInterfaceName,
Flags: net.FlagUp,
MTU: contVeth.Attrs().MTU,
MTU: link.Attrs().MTU,
},
Mode: netlink.TUNTAP_MODE_TAP,
}
Expand All @@ -587,7 +645,7 @@ func SetupContainerSideNetwork(info *cnicurrent.Result) (*ContainerSideNetwork,
}

containerBridgeName := fmt.Sprintf(containerBridgeNameTemplate, i)
br, err := SetupBridge(containerBridgeName, []netlink.Link{contVeth, tap})
br, err := SetupBridge(containerBridgeName, []netlink.Link{link, tap})
if err != nil {
return nil, fmt.Errorf("failed to create bridge: %v", err)
}
Expand All @@ -597,7 +655,7 @@ func SetupContainerSideNetwork(info *cnicurrent.Result) (*ContainerSideNetwork,
}

// Add ebtables DHCP blocking rules
if err := updateEbTables(contVeth.Attrs().Name, "-A"); err != nil {
if err := updateEbTables(link.Attrs().Name, "-A"); err != nil {
return nil, err
}

Expand Down Expand Up @@ -695,14 +753,14 @@ func (csn *ContainerSideNetwork) Teardown() error {
for _, f := range csn.TapFiles {
f.Close()
}
contVeths, err := GetContainerInterfaces(csn.Result)
contLinks, err := GetContainerLinks(csn.Result.Interfaces)
if err != nil {
return err
}

for i, contVeth := range contVeths {
for i, contLink := range contLinks {
// Remove ebtables DHCP rules
if err := updateEbTables(contVeth.Attrs().Name, "-D"); err != nil {
if err := updateEbTables(contLink.Attrs().Name, "-D"); err != nil {
return nil
}

Expand All @@ -722,7 +780,7 @@ func (csn *ContainerSideNetwork) Teardown() error {
return err
}

if err := TeardownBridge(br, []netlink.Link{contVeth, tap}); err != nil {
if err := TeardownBridge(br, []netlink.Link{contLink, tap}); err != nil {
return err
}

Expand All @@ -738,11 +796,11 @@ func (csn *ContainerSideNetwork) Teardown() error {
return err
}

if err := SetHardwareAddr(contVeth, csn.HardwareAddrs[i]); err != nil {
if err := SetHardwareAddr(contLink, csn.HardwareAddrs[i]); err != nil {
return err
}

if err := ConfigureLink(contVeth, i, csn.Result); err != nil {
if err := ConfigureLink(contLink, i, csn.Result); err != nil {
return err
}
}
Expand Down
14 changes: 4 additions & 10 deletions pkg/nettools/nettools_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,9 +346,9 @@ func TestExtractLinkInfo(t *testing.T) {
})
}

func verifyContainerSideNetwork(t *testing.T, origContVeth netlink.Link, info *cnicurrent.Result) {
func verifyContainerSideNetwork(t *testing.T, origContVeth netlink.Link) {
origHwAddr := origContVeth.Attrs().HardwareAddr
csn, err := SetupContainerSideNetwork(info)
csn, err := SetupContainerSideNetwork(&expectedExtractedLinkInfo)
if err != nil {
log.Panicf("failed to set up container side network: %v", err)
}
Expand Down Expand Up @@ -389,24 +389,18 @@ func verifyContainerSideNetwork(t *testing.T, origContVeth netlink.Link, info *c
}
}

func TestSetUpContainerSideNetwork(t *testing.T) {
withFakeCNIVeth(t, func(hostNS, contNS ns.NetNS, origHostVeth, origContVeth netlink.Link) {
verifyContainerSideNetwork(t, origContVeth, nil)
})
}

func TestSetUpContainerSideNetworkWithInfo(t *testing.T) {
withFakeCNIVeth(t, func(hostNS, contNS ns.NetNS, origHostVeth, origContVeth netlink.Link) {
if err := StripLink(origContVeth); err != nil {
log.Panicf("StripLink() failed: %v", err)
}
verifyContainerSideNetwork(t, origContVeth, &expectedExtractedLinkInfo)
verifyContainerSideNetwork(t, origContVeth)
})
}

func TestLoopbackInterface(t *testing.T) {
withFakeCNIVeth(t, func(hostNS, contNS ns.NetNS, origHostVeth, origContVeth netlink.Link) {
verifyContainerSideNetwork(t, origContVeth, nil)
verifyContainerSideNetwork(t, origContVeth)
if out, err := exec.Command("ping", "-c", "1", "127.0.0.1").CombinedOutput(); err != nil {
log.Panicf("ping 127.0.0.1 failed:\n%s", out)
}
Expand Down
43 changes: 7 additions & 36 deletions pkg/tapmanager/tapfdsource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,13 @@ func (s *TapFDSource) GetFDs(key string, data []byte) ([]int, []byte, error) {
}
glog.V(3).Infof("CNI configuration for pod %s (%s): %s", pnd.PodName, pnd.PodId, spew.Sdump(netConfig))

if netConfig == nil {
netConfig = &cnicurrent.Result{}
}
if err := nettools.ValidateAndfixCNIResult(netConfig); err != nil {
return nil, nil, fmt.Errorf("error in fixing cni configuration: %v", err)
}

if payload.Description.DNS != nil {
netConfig.DNS.Nameservers = pnd.DNS.Nameservers
netConfig.DNS.Search = pnd.DNS.Search
Expand Down Expand Up @@ -206,13 +213,6 @@ func (s *TapFDSource) GetFDs(key string, data []byte) ([]int, []byte, error) {
return err
}

// NOTE: older CNI plugins don't include the hardware address
// in Result, but it's needed for Cloud-Init based
// network setup, so we add it here if it's missing.
// Also, some of the plugins may skip adding routes
// to the CNI result, so we must add them, too
fixCNIResult(netConfig, csn)

dhcpServer, err = dhcp.NewServer(csn.Result)
if err != nil {
return err
Expand Down Expand Up @@ -322,35 +322,6 @@ func (s *TapFDSource) GetInfo(key string) ([]byte, error) {
return data, nil
}

func fixCNIResult(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) {
// If there's no interface info in netConfig, we can assume that we're dealing
// with an old-style CNI plugin which only supports a single network interface
if len(netConfig.Interfaces) > 0 {
return
}

// TODO: get real interface name from links scan for matching mac add
// instead of generating fake one
for i, mac := range csn.HardwareAddrs {
name := fmt.Sprintf("cni%d", i)
iface := &cnicurrent.Interface{
Name: name,
Mac: mac.String(),
}
netConfig.Interfaces = append(netConfig.Interfaces, iface)
}

// TODO: scan interfaces for matching ip addresses instead of setting first interface
// as the target for ip address
for _, IP := range netConfig.IPs {
IP.Interface = 0
}

if len(netConfig.Routes) == 0 {
netConfig.Routes = csn.Result.Routes
}
}

func netmaskForCalico() net.IPMask {
n := calicoDefaultSubnet
subnetStr := os.Getenv(calicoSubnetVar)
Expand Down

0 comments on commit 7ca71a3

Please sign in to comment.