diff --git a/cni/plugins/main/multi-nic/ipvlan.go b/cni/plugins/main/multi-nic/ipvlan.go index 34db6732..63c12b6c 100644 --- a/cni/plugins/main/multi-nic/ipvlan.go +++ b/cni/plugins/main/multi-nic/ipvlan.go @@ -11,6 +11,7 @@ import ( "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/utils" "github.com/vishvananda/netlink" ) @@ -40,6 +41,17 @@ func loadIPVANConf(bytes []byte, ifName string, n *NetConf, ipConfigs []*current return } + var ipamMap map[string][]byte + + if n.IPAM.Type == MultiConfigIPAMType { + var err error + ipamMap, err = getMultiIPAMConfigBytes(bytes) + if err != nil { + ipamMap = nil + utils.Logger.Debug(fmt.Sprintf("getMultiIPAMConfigBytes failed: %v", err)) + } + } + // interfaces are orderly assigned from interface set for index, masterName := range n.Masters { if masterName == "" { @@ -62,9 +74,16 @@ func loadIPVANConf(bytes []byte, ifName string, n *NetConf, ipConfigs []*current return } - if n.IsMultiNICIPAM { - // multi-NIC IPAM config + if len(ipConfigs) > 0 { + // no need to call ipam due to static ip confBytes, multiPathRoutes = injectMultiNicIPAM(confBytes, bytes, ipConfigs, index) + } else if ipamMap != nil { + if ipamBytes, found := ipamMap[masterName]; found { + confBytes, multiPathRoutes = replaceSingleNicIPAMWithMultiConfig(confBytes, bytes, ipamBytes) + } else { + utils.Logger.Debug(fmt.Sprintf("Multi-config IPAM has no definition of %s", masterName)) + continue + } } else { confBytes, multiPathRoutes = injectSingleNicIPAM(confBytes, bytes) } diff --git a/cni/plugins/main/multi-nic/multi-nic.go b/cni/plugins/main/multi-nic/multi-nic.go index dea82762..8897b038 100644 --- a/cni/plugins/main/multi-nic/multi-nic.go +++ b/cni/plugins/main/multi-nic/multi-nic.go @@ -125,6 +125,30 @@ func cmdAdd(args *skel.CmdArgs) error { if len(result.IPs) == 0 { return fmt.Errorf("IPAM plugin returned missing IP config %v", string(injectedStdIn)) } + } else if !haveResult && n.IPAM.Type == MultiConfigIPAMType { + var ips []string + ips, args.Args = getStaticIPs(args.Args) + for index, ipnet := range ips { + ipVal, reservedIP, err := net.ParseCIDR(ipnet) + if err != nil { + utils.Logger.Debug(fmt.Sprintf("failed to parse static IP %s: %v", ipnet, err)) + return err + } + reservedIP.IP = ipVal + ipConf := ¤t.IPConfig{ + Address: *reservedIP, + Interface: current.Int(index), + } + result.IPs = append(result.IPs, ipConf) + } + if len(n.Masters) == 0 { + ipamObject, err := getMultiIPAMConfig(args.StdinData) + if err == nil && ipamObject.IPAM.Args != nil { + for masterName, _ := range ipamObject.IPAM.Args { + n.Masters = append(n.Masters, masterName) + } + } + } } // get device config and apply diff --git a/cni/plugins/main/multi-nic/multi-nic_test.go b/cni/plugins/main/multi-nic/multi-nic_test.go index 677d532c..e6b0ec24 100644 --- a/cni/plugins/main/multi-nic/multi-nic_test.go +++ b/cni/plugins/main/multi-nic/multi-nic_test.go @@ -533,6 +533,71 @@ var _ = Describe("Operations", func() { multiPathTest(ver, singleNICIPAMWithMultiPath, fullNets, POOL_MASTER_NAMES) }) + It(fmt.Sprintf("[%s] check multi-config IPAM", ver), func() { + conf, n := getSampleMultiIPAMConfig(ver, POOL_MASTER_NAMES, masterNets) + confBytesArray, _, err := loadIPVANConf([]byte(conf), "net1", n, []*types100.IPConfig{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(confBytesArray)).To(Equal(len(POOL_MASTER_NAMES))) + for index, confBytes := range confBytesArray { + log.Printf("%s", confBytes) + confObj := &IPVLANTypeNetConf{} + err = json.Unmarshal(confBytes, confObj) + Expect(err).NotTo(HaveOccurred()) + Expect(confObj.IPAM.Type).To(Equal("whereabouts")) + ipamObject := &IPAMExtract{} + err = json.Unmarshal(confBytes, ipamObject) + Expect(err).NotTo(HaveOccurred()) + Expect(ipamObject.IPAM["range"].(string)).To(Equal(POOL_IP_ADDRESSES[index])) + Expect(ipamObject.IPAM["network_name"].(string)).To(Equal(POOL_MASTER_NAMES[index])) + } + }) + + It(fmt.Sprintf("[%s] check multi-config IPAM with static IP", ver), func() { + var ips []*types100.IPConfig + for _, podIP := range POOL_IP_ADDRESSES { + ipVal, ipnet, err := net.ParseCIDR(podIP) + ipnet.IP = ipVal + Expect(err).NotTo(HaveOccurred()) + podIPConfig := &types100.IPConfig{Address: *ipnet} + ips = append(ips, podIPConfig) + } + conf, n := getSampleMultiIPAMConfig(ver, POOL_MASTER_NAMES, masterNets) + confBytesArray, _, err := loadIPVANConf([]byte(conf), "net1", n, ips) + Expect(err).NotTo(HaveOccurred()) + Expect(len(confBytesArray)).To(Equal(len(POOL_MASTER_NAMES))) + for index, confBytes := range confBytesArray { + log.Printf("%s", confBytes) + confObj := &IPVLANTypeNetConf{} + err = json.Unmarshal(confBytes, confObj) + Expect(err).NotTo(HaveOccurred()) + Expect(confObj.IPAM.Type).To(Equal("static")) + ipamObject := &IPAMExtract{} + err = json.Unmarshal(confBytes, ipamObject) + Expect(err).NotTo(HaveOccurred()) + addresses := ipamObject.IPAM["addresses"].([]interface{}) + Expect(len(addresses)).To(Equal(1)) + addressMap := addresses[0].(map[string]interface{}) + Expect(addressMap["address"].(string)).To(Equal(POOL_IP_ADDRESSES[index])) + } + }) + + It(fmt.Sprintf("[%s] check getStaticIPs", ver), func() { + var ips []string + joinedIPs := strings.Join(POOL_IP_ADDRESSES, ",") + argStr := fmt.Sprintf("POD_NAME=a;IP=%s", joinedIPs) + ips, argStr = getStaticIPs(argStr) + for index, podIP := range ips { + Expect(podIP).To(Equal(POOL_IP_ADDRESSES[index])) + } + Expect(argStr).To(Equal("POD_NAME=a")) + + argStr = fmt.Sprintf("POD_NAME=a;IP=%s;SOME_ARG=b", joinedIPs) + ips, argStr = getStaticIPs(argStr) + for index, podIP := range ips { + Expect(podIP).To(Equal(POOL_IP_ADDRESSES[index])) + } + Expect(argStr).To(Equal("POD_NAME=a;SOME_ARG=b")) + }) } }) @@ -583,6 +648,48 @@ func getAwsIpvlanConfig(ver, masterNets string) ([]byte, *NetConf) { return conf, n } +func getSampleMultiIPAMConfig(ver string, masterNames []string, masterNets string) ([]byte, *NetConf) { + ipamArgs := "" + for index, masterName := range masterNames { + if index > 0 { + ipamArgs += "," + } + ipamArgs += fmt.Sprintf(` + "%s": {"range": "%s"} + `, masterName, POOL_IP_ADDRESSES[index]) + } + confStr := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "multi-nic-sample", + "type": "multi-nic", + "plugin": { + "cniVersion": "0.3.0", + "type": "ipvlan", + "mode": "l2" + }, + "vlanMode": "l2", + "ipam": { + "type": "multi-config", + "ipam_type": "whereabouts", + "args": { + %s + } + }, + "daemonIP": "%s", + "daemonPort": %d, + "subnet": "192.168.0.0/16", + "masterNets": %s + }`, ver, ipamArgs, BRIDGE_HOST_IP, daemonPort, masterNets) + log.Printf("%s", confStr) + conf := []byte(confStr) + n := &NetConf{} + err := json.Unmarshal(conf, n) + Expect(err).NotTo(HaveOccurred()) + n.DeviceIDs = POOL_MASTER_NAMES + n.Masters = POOL_MASTER_NAMES + return conf, n +} + func closeServer(srv *http.Server, httpServerExitDone *sync.WaitGroup) { if err := srv.Shutdown(context.TODO()); err != nil { panic(err) diff --git a/cni/plugins/main/multi-nic/utils.go b/cni/plugins/main/multi-nic/utils.go index 9066b3e5..af451395 100644 --- a/cni/plugins/main/multi-nic/utils.go +++ b/cni/plugins/main/multi-nic/utils.go @@ -15,9 +15,31 @@ import ( "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/utils" "github.com/vishvananda/netlink" ) +const ( + MultiConfigIPAMType = "multi-config" + WhereaboutsIPAMType = "whereabouts" +) + +type IPAMExtract struct { + IPAM map[string]interface{} `json:"ipam"` +} + +type MultiIPAMConfig struct { + Name string + Type string `json:"type"` + IpamType string `json:"ipam_type"` + Args map[string]map[string]interface{} `json:"args"` + Routes []*types.Route `json:"routes"` +} + +type MultiIPAMExtract struct { + IPAM MultiIPAMConfig `json:"ipam"` +} + // getPodInfo extracts pod Name and Namespace from cniArgs func getPodInfo(cniArgs string) (string, string) { splits := strings.Split(cniArgs, ";") @@ -33,6 +55,57 @@ func getPodInfo(cniArgs string) (string, string) { return podName, podNamespace } +// getStaticIPs extracts static ip from cniArgs +// reference: https://github.com/k8snetworkplumbingwg/multus-cni/blob/e2e8cfb677e8cf5352f737b5004effed7518d71a/pkg/multus/multus.go#L324 +func getStaticIPs(cniArgs string) ([]string, string) { + splits := strings.Split(cniArgs, ";") + for index, split := range splits { + if strings.HasPrefix(split, "IP=") { + ipsStr := strings.TrimPrefix(split, "IP=") + newItems := splits[0:index] + if index < len(split)-1 { + newItems = append(newItems, splits[index+1:len(splits)]...) + } + newCniArgs := strings.Join(newItems, ";") + return strings.Split(ipsStr, ","), newCniArgs + } + } + return []string{}, cniArgs +} + +func getMultiIPAMConfig(multiNicConfBytes []byte) (*MultiIPAMExtract, error) { + ipamObject := &MultiIPAMExtract{} + err := json.Unmarshal(multiNicConfBytes, ipamObject) + return ipamObject, err + +} + +func getMultiIPAMConfigBytes(multiNicConfBytes []byte) (map[string][]byte, error) { + multiIPAMConfigBytes := make(map[string][]byte) + ipamObject, err := getMultiIPAMConfig(multiNicConfBytes) + if err == nil && ipamObject.IPAM.Args != nil { + for masterName, singleIPAMObject := range ipamObject.IPAM.Args { + singleIPAMConfBytes := make(map[string]interface{}) + if singleIPAMObject == nil { + utils.Logger.Debug(fmt.Sprintf("ipamObject.IPAM.Args not defined on %s", masterName)) + continue + } + // set type + singleIPAMConfBytes["type"] = ipamObject.IPAM.IpamType + if ipamObject.IPAM.IpamType == WhereaboutsIPAMType { + singleIPAMConfBytes["network_name"] = masterName + } + // set args + for k, v := range singleIPAMObject { + singleIPAMConfBytes[k] = v + } + ipamBytes, _ := json.Marshal(singleIPAMConfBytes) + multiIPAMConfigBytes[masterName] = ipamBytes + } + } + return multiIPAMConfigBytes, err +} + // injectIPAM injects ipam bytes to config func injectMultiNicIPAM(singleNicConfBytes, multiNicConfBytes []byte, ipConfigs []*current.IPConfig, ipIndex int) ([]byte, map[string][]*netlink.NexthopInfo) { var ipConfig *current.IPConfig @@ -46,11 +119,17 @@ func injectSingleNicIPAM(singleNicConfBytes []byte, multiNicConfBytes []byte) ([ return replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes) } -type IPAMExtract struct { - IPAM map[string]interface{} `json:"ipam"` +func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) { + ipamObject := &IPAMExtract{} + err := json.Unmarshal(multiNicConfBytes, ipamObject) + if err == nil { + ipamBytes, _ := json.Marshal(ipamObject.IPAM) + return replaceSingleNicIPAMWithMultiConfig(singleNicConfBytes, multiNicConfBytes, ipamBytes) + } + return singleNicConfBytes, nil } -func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) { +func replaceSingleNicIPAMWithMultiConfig(singleNicConfBytes, multiNicConfBytes, ipamBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) { confStr := string(singleNicConfBytes) ipamObject := &IPAMExtract{} err := json.Unmarshal(multiNicConfBytes, ipamObject) @@ -60,7 +139,6 @@ func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte, ipamObject.IPAM["routes"] = nonMultiPathRoutes } - ipamBytes, _ := json.Marshal(ipamObject.IPAM) singleIPAM := fmt.Sprintf("\"ipam\":%s", string(ipamBytes)) injectedStr := strings.ReplaceAll(confStr, "\"ipam\":{}", singleIPAM) return []byte(injectedStr), multiPathRoutes diff --git a/config/samples/multinicnetwork/ipvlanl2_multiconfig.yaml b/config/samples/multinicnetwork/ipvlanl2_multiconfig.yaml new file mode 100644 index 00000000..5805b07f --- /dev/null +++ b/config/samples/multinicnetwork/ipvlanl2_multiconfig.yaml @@ -0,0 +1,25 @@ +apiVersion: multinic.fms.io/v1 +kind: MultiNicNetwork +metadata: + name: multinic-multiconfig +spec: + subnet: "" + ipam: | + { + "type": "multi-config", + "ipam_type": "whereabouts", + "args": { + "p0": { + "range": "192.168.0.0/18" + }, + "p1": { + "range": "192.168.64.0/18" + } + } + } + multiNICIPAM: false + plugin: + cniVersion: "0.3.0" + type: ipvlan + args: + mode: l2 \ No newline at end of file