add multi-config and handle static ips

Signed-off-by: Sunyanan Choochotkaew <[email protected]>

cni/plugins/main/multi-nic/ipvlan.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 	current "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/containernetworking/plugins/pkg/utils"
 	"github.com/vishvananda/netlink"
 )
 
@@ -40,6 +41,17 @@ func loadIPVANConf(bytes []byte, ifName string, n *NetConf, ipConfigs []*current
 		return
 	}
 
+	var ipamMap map[string][]byte
+
+	if n.IPAM.Type == MultiConfigIPAMType {
+		var err error
+		ipamMap, err = getMultiIPAMConfigBytes(bytes)
+		if err != nil {
+			ipamMap = nil
+			utils.Logger.Debug(fmt.Sprintf("getMultiIPAMConfigBytes failed: %v", err))
+		}
+	}
+
 	// interfaces are orderly assigned from interface set
 	for index, masterName := range n.Masters {
 		if masterName == "" {
@@ -62,9 +74,16 @@ func loadIPVANConf(bytes []byte, ifName string, n *NetConf, ipConfigs []*current
 			return
 		}
 
-		if n.IsMultiNICIPAM {
-			// multi-NIC IPAM config
+		if len(ipConfigs) > 0 {
+			// no need to call ipam due to static ip
 			confBytes, multiPathRoutes = injectMultiNicIPAM(confBytes, bytes, ipConfigs, index)
+		} else if ipamMap != nil {
+			if ipamBytes, found := ipamMap[masterName]; found {
+				confBytes, multiPathRoutes = replaceSingleNicIPAMWithMultiConfig(confBytes, bytes, ipamBytes)
+			} else {
+				utils.Logger.Debug(fmt.Sprintf("Multi-config IPAM has no definition of %s", masterName))
+				continue
+			}
 		} else {
 			confBytes, multiPathRoutes = injectSingleNicIPAM(confBytes, bytes)
 		}

cni/plugins/main/multi-nic/multi-nic.go

@@ -125,6 +125,30 @@ func cmdAdd(args *skel.CmdArgs) error {
 		if len(result.IPs) == 0 {
 			return fmt.Errorf("IPAM plugin returned missing IP config %v", string(injectedStdIn))
 		}
+	} else if !haveResult && n.IPAM.Type == MultiConfigIPAMType {
+		var ips []string
+		ips, args.Args = getStaticIPs(args.Args)
+		for index, ipnet := range ips {
+			ipVal, reservedIP, err := net.ParseCIDR(ipnet)
+			if err != nil {
+				utils.Logger.Debug(fmt.Sprintf("failed to parse static IP %s: %v", ipnet, err))
+				return err
+			}
+			reservedIP.IP = ipVal
+			ipConf := &current.IPConfig{
+				Address:   *reservedIP,
+				Interface: current.Int(index),
+			}
+			result.IPs = append(result.IPs, ipConf)
+		}
+		if len(n.Masters) == 0 {
+			ipamObject, err := getMultiIPAMConfig(args.StdinData)
+			if err == nil && ipamObject.IPAM.Args != nil {
+				for masterName, _ := range ipamObject.IPAM.Args {
+					n.Masters = append(n.Masters, masterName)
+				}
+			}
+		}
 	}
 
 	// get device config and apply

cni/plugins/main/multi-nic/multi-nic_test.go

@@ -533,6 +533,71 @@ var _ = Describe("Operations", func() {
 			multiPathTest(ver, singleNICIPAMWithMultiPath, fullNets, POOL_MASTER_NAMES)
 		})
 
+		It(fmt.Sprintf("[%s] check multi-config IPAM", ver), func() {
+			conf, n := getSampleMultiIPAMConfig(ver, POOL_MASTER_NAMES, masterNets)
+			confBytesArray, _, err := loadIPVANConf([]byte(conf), "net1", n, []*types100.IPConfig{})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(confBytesArray)).To(Equal(len(POOL_MASTER_NAMES)))
+			for index, confBytes := range confBytesArray {
+				log.Printf("%s", confBytes)
+				confObj := &IPVLANTypeNetConf{}
+				err = json.Unmarshal(confBytes, confObj)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(confObj.IPAM.Type).To(Equal("whereabouts"))
+				ipamObject := &IPAMExtract{}
+				err = json.Unmarshal(confBytes, ipamObject)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(ipamObject.IPAM["range"].(string)).To(Equal(POOL_IP_ADDRESSES[index]))
+				Expect(ipamObject.IPAM["network_name"].(string)).To(Equal(POOL_MASTER_NAMES[index]))
+			}
+		})
+
+		It(fmt.Sprintf("[%s] check multi-config IPAM with static IP", ver), func() {
+			var ips []*types100.IPConfig
+			for _, podIP := range POOL_IP_ADDRESSES {
+				ipVal, ipnet, err := net.ParseCIDR(podIP)
+				ipnet.IP = ipVal
+				Expect(err).NotTo(HaveOccurred())
+				podIPConfig := &types100.IPConfig{Address: *ipnet}
+				ips = append(ips, podIPConfig)
+			}
+			conf, n := getSampleMultiIPAMConfig(ver, POOL_MASTER_NAMES, masterNets)
+			confBytesArray, _, err := loadIPVANConf([]byte(conf), "net1", n, ips)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(confBytesArray)).To(Equal(len(POOL_MASTER_NAMES)))
+			for index, confBytes := range confBytesArray {
+				log.Printf("%s", confBytes)
+				confObj := &IPVLANTypeNetConf{}
+				err = json.Unmarshal(confBytes, confObj)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(confObj.IPAM.Type).To(Equal("static"))
+				ipamObject := &IPAMExtract{}
+				err = json.Unmarshal(confBytes, ipamObject)
+				Expect(err).NotTo(HaveOccurred())
+				addresses := ipamObject.IPAM["addresses"].([]interface{})
+				Expect(len(addresses)).To(Equal(1))
+				addressMap := addresses[0].(map[string]interface{})
+				Expect(addressMap["address"].(string)).To(Equal(POOL_IP_ADDRESSES[index]))
+			}
+		})
+
+		It(fmt.Sprintf("[%s] check getStaticIPs", ver), func() {
+			var ips []string
+			joinedIPs := strings.Join(POOL_IP_ADDRESSES, ",")
+			argStr := fmt.Sprintf("POD_NAME=a;IP=%s", joinedIPs)
+			ips, argStr = getStaticIPs(argStr)
+			for index, podIP := range ips {
+				Expect(podIP).To(Equal(POOL_IP_ADDRESSES[index]))
+			}
+			Expect(argStr).To(Equal("POD_NAME=a"))
+
+			argStr = fmt.Sprintf("POD_NAME=a;IP=%s;SOME_ARG=b", joinedIPs)
+			ips, argStr = getStaticIPs(argStr)
+			for index, podIP := range ips {
+				Expect(podIP).To(Equal(POOL_IP_ADDRESSES[index]))
+			}
+			Expect(argStr).To(Equal("POD_NAME=a;SOME_ARG=b"))
+		})
 	}
 })
 
@@ -583,6 +648,48 @@ func getAwsIpvlanConfig(ver, masterNets string) ([]byte, *NetConf) {
 	return conf, n
 }
 
+func getSampleMultiIPAMConfig(ver string, masterNames []string, masterNets string) ([]byte, *NetConf) {
+	ipamArgs := ""
+	for index, masterName := range masterNames {
+		if index > 0 {
+			ipamArgs += ","
+		}
+		ipamArgs += fmt.Sprintf(`
+		"%s": {"range": "%s"}
+		`, masterName, POOL_IP_ADDRESSES[index])
+	}
+	confStr := fmt.Sprintf(`{ 
+		"cniVersion": "%s", 
+		"name": "multi-nic-sample",
+		"type": "multi-nic",
+		"plugin": {
+			"cniVersion": "0.3.0",
+			"type": "ipvlan",
+			"mode": "l2"
+		},
+		"vlanMode": "l2",
+		"ipam": {
+			"type": "multi-config",
+			"ipam_type": "whereabouts",
+			"args": { 
+				%s
+			}
+		},
+		"daemonIP": "%s",
+		"daemonPort": %d,
+		"subnet": "192.168.0.0/16",
+		"masterNets": %s
+		}`, ver, ipamArgs, BRIDGE_HOST_IP, daemonPort, masterNets)
+	log.Printf("%s", confStr)
+	conf := []byte(confStr)
+	n := &NetConf{}
+	err := json.Unmarshal(conf, n)
+	Expect(err).NotTo(HaveOccurred())
+	n.DeviceIDs = POOL_MASTER_NAMES
+	n.Masters = POOL_MASTER_NAMES
+	return conf, n
+}
+
 func closeServer(srv *http.Server, httpServerExitDone *sync.WaitGroup) {
 	if err := srv.Shutdown(context.TODO()); err != nil {
 		panic(err)

cni/plugins/main/multi-nic/utils.go

@@ -15,9 +15,31 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 	current "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/containernetworking/plugins/pkg/utils"
 	"github.com/vishvananda/netlink"
 )
 
+const (
+	MultiConfigIPAMType = "multi-config"
+	WhereaboutsIPAMType = "whereabouts"
+)
+
+type IPAMExtract struct {
+	IPAM map[string]interface{} `json:"ipam"`
+}
+
+type MultiIPAMConfig struct {
+	Name     string
+	Type     string                            `json:"type"`
+	IpamType string                            `json:"ipam_type"`
+	Args     map[string]map[string]interface{} `json:"args"`
+	Routes   []*types.Route                    `json:"routes"`
+}
+
+type MultiIPAMExtract struct {
+	IPAM MultiIPAMConfig `json:"ipam"`
+}
+
 // getPodInfo extracts pod Name and Namespace from cniArgs
 func getPodInfo(cniArgs string) (string, string) {
 	splits := strings.Split(cniArgs, ";")
@@ -33,6 +55,57 @@ func getPodInfo(cniArgs string) (string, string) {
 	return podName, podNamespace
 }
 
+// getStaticIPs extracts static ip from cniArgs
+// reference: https://github.com/k8snetworkplumbingwg/multus-cni/blob/e2e8cfb677e8cf5352f737b5004effed7518d71a/pkg/multus/multus.go#L324
+func getStaticIPs(cniArgs string) ([]string, string) {
+	splits := strings.Split(cniArgs, ";")
+	for index, split := range splits {
+		if strings.HasPrefix(split, "IP=") {
+			ipsStr := strings.TrimPrefix(split, "IP=")
+			newItems := splits[0:index]
+			if index < len(split)-1 {
+				newItems = append(newItems, splits[index+1:len(splits)]...)
+			}
+			newCniArgs := strings.Join(newItems, ";")
+			return strings.Split(ipsStr, ","), newCniArgs
+		}
+	}
+	return []string{}, cniArgs
+}
+
+func getMultiIPAMConfig(multiNicConfBytes []byte) (*MultiIPAMExtract, error) {
+	ipamObject := &MultiIPAMExtract{}
+	err := json.Unmarshal(multiNicConfBytes, ipamObject)
+	return ipamObject, err
+
+}
+
+func getMultiIPAMConfigBytes(multiNicConfBytes []byte) (map[string][]byte, error) {
+	multiIPAMConfigBytes := make(map[string][]byte)
+	ipamObject, err := getMultiIPAMConfig(multiNicConfBytes)
+	if err == nil && ipamObject.IPAM.Args != nil {
+		for masterName, singleIPAMObject := range ipamObject.IPAM.Args {
+			singleIPAMConfBytes := make(map[string]interface{})
+			if singleIPAMObject == nil {
+				utils.Logger.Debug(fmt.Sprintf("ipamObject.IPAM.Args not defined on %s", masterName))
+				continue
+			}
+			// set type
+			singleIPAMConfBytes["type"] = ipamObject.IPAM.IpamType
+			if ipamObject.IPAM.IpamType == WhereaboutsIPAMType {
+				singleIPAMConfBytes["network_name"] = masterName
+			}
+			// set args
+			for k, v := range singleIPAMObject {
+				singleIPAMConfBytes[k] = v
+			}
+			ipamBytes, _ := json.Marshal(singleIPAMConfBytes)
+			multiIPAMConfigBytes[masterName] = ipamBytes
+		}
+	}
+	return multiIPAMConfigBytes, err
+}
+
 // injectIPAM injects ipam bytes to config
 func injectMultiNicIPAM(singleNicConfBytes, multiNicConfBytes []byte, ipConfigs []*current.IPConfig, ipIndex int) ([]byte, map[string][]*netlink.NexthopInfo) {
 	var ipConfig *current.IPConfig
@@ -46,11 +119,17 @@ func injectSingleNicIPAM(singleNicConfBytes []byte, multiNicConfBytes []byte) ([
 	return replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes)
 }
 
-type IPAMExtract struct {
-	IPAM map[string]interface{} `json:"ipam"`
+func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) {
+	ipamObject := &IPAMExtract{}
+	err := json.Unmarshal(multiNicConfBytes, ipamObject)
+	if err == nil {
+		ipamBytes, _ := json.Marshal(ipamObject.IPAM)
+		return replaceSingleNicIPAMWithMultiConfig(singleNicConfBytes, multiNicConfBytes, ipamBytes)
+	}
+	return singleNicConfBytes, nil
 }
 
-func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) {
+func replaceSingleNicIPAMWithMultiConfig(singleNicConfBytes, multiNicConfBytes, ipamBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) {
 	confStr := string(singleNicConfBytes)
 	ipamObject := &IPAMExtract{}
 	err := json.Unmarshal(multiNicConfBytes, ipamObject)
@@ -60,7 +139,6 @@ func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte,
 			ipamObject.IPAM["routes"] = nonMultiPathRoutes
 		}
 
-		ipamBytes, _ := json.Marshal(ipamObject.IPAM)
 		singleIPAM := fmt.Sprintf("\"ipam\":%s", string(ipamBytes))
 		injectedStr := strings.ReplaceAll(confStr, "\"ipam\":{}", singleIPAM)
 		return []byte(injectedStr), multiPathRoutes

config/samples/multinicnetwork/ipvlanl2_multiconfig.yaml

@@ -0,0 +1,25 @@
+apiVersion: multinic.fms.io/v1
+kind: MultiNicNetwork
+metadata:
+  name: multinic-multiconfig
+spec:
+  subnet: ""
+  ipam: |
+    {
+      "type": "multi-config",
+      "ipam_type": "whereabouts",
+      "args": {
+        "p0": {
+          "range": "192.168.0.0/18"
+        },
+        "p1": {
+          "range": "192.168.64.0/18"
+        }
+      }
+    }
+  multiNICIPAM: false
+  plugin:
+    cniVersion: "0.3.0"
+    type: ipvlan
+    args: 
+      mode: l2