kubeovn · zhangzujian · Oct 8, 2023 · Oct 8, 2023 · Oct 8, 2023
diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,7 @@ broker-info.subm
 broker-info.subm.*
 broker-info-internal.subm
 yamls/clab-bgp.yaml
+yamls/clab-bgp-ha.yaml
 kube-ovn.tar
 vpc-nat-gateway.tar
 image-amd64.tar

diff --git a/Makefile b/Makefile
@@ -416,6 +416,19 @@ kind-init-bgp: kind-clean-bgp kind-init
 		-v $(CURDIR)/yamls/clab-bgp.yaml:/clab.yaml \
 		$(CLAB_IMAGE) clab deploy -t /clab.yaml
 
+.PHONY: kind-init-bgp-ha
+kind-init-bgp-ha: kind-clean-bgp kind-init
+	kube_ovn_version=$(VERSION) j2 yamls/clab-bgp-ha.yaml.j2 -o yamls/clab-bgp-ha.yaml
+	docker run --rm --privileged \
+		--name kube-ovn-bgp \
+		--network host \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		-v /var/run/netns:/var/run/netns \
+		-v /var/lib/docker/containers:/var/lib/docker/containers \
+		--pid=host \
+		-v $(CURDIR)/yamls/clab-bgp-ha.yaml:/clab.yaml \
+		$(CLAB_IMAGE) clab deploy -t /clab.yaml
+
 .PHONY: kind-load-image
 kind-load-image:
 	$(call kind_load_image,kube-ovn,$(REGISTRY)/kube-ovn:$(VERSION))
@@ -792,6 +805,20 @@ kind-install-bgp: kind-install
 		-e 's/--cluster-as=.*/--cluster-as=65002/' yamls/speaker.yaml | \
 		kubectl apply -f -
 	kubectl -n kube-system rollout status ds kube-ovn-speaker --timeout 60s
+	docker exec clab-bgp-router vtysh -c "show ip route bgp"
+
+.PHONY: kind-install-bgp-ha
+kind-install-bgp-ha: kind-install
+	kubectl label node --all ovn.kubernetes.io/bgp=true
+	kubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=local
+	sed -e 's#image: .*#image: $(REGISTRY)/kube-ovn:$(VERSION)#' \
+		-e 's/--neighbor-address=.*/--neighbor-address=10.0.1.1,10.0.1.2/' \
+		-e 's/--neighbor-as=.*/--neighbor-as=65001/' \
+		-e 's/--cluster-as=.*/--cluster-as=65002/' yamls/speaker.yaml | \
+		kubectl apply -f -
+	kubectl -n kube-system rollout status ds kube-ovn-speaker --timeout 60s
+	docker exec clab-bgp-router-1 vtysh -c "show ip route bgp"
+	docker exec clab-bgp-router-2 vtysh -c "show ip route bgp"
 
 .PHONY: kind-install-deepflow
 kind-install-deepflow: kind-install
@@ -845,8 +872,7 @@ kind-clean-ovn-submariner: kind-clean
 	kind delete cluster --name=kube-ovn1
 
 .PHONY: kind-clean-bgp
-kind-clean-bgp:
-	$(call docker_rm_container,kube-ovn-bgp)
+kind-clean-bgp: kind-clean-bgp-ha
 	kube_ovn_version=$(VERSION) j2 yamls/clab-bgp.yaml.j2 -o yamls/clab-bgp.yaml
 	docker run --rm --privileged \
 		--name kube-ovn-bgp \
@@ -859,6 +885,20 @@ kind-clean-bgp:
 		$(CLAB_IMAGE) clab destroy -t /clab.yaml
 	@$(MAKE) kind-clean
 
+.PHONY: kind-clean-bgp-ha
+kind-clean-bgp-ha:
+	kube_ovn_version=$(VERSION) j2 yamls/clab-bgp-ha.yaml.j2 -o yamls/clab-bgp-ha.yaml
+	docker run --rm --privileged \
+		--name kube-ovn-bgp \
+		--network host \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		-v /var/run/netns:/var/run/netns \
+		-v /var/lib/docker/containers:/var/lib/docker/containers \
+		--pid=host \
+		-v $(CURDIR)/yamls/clab-bgp-ha.yaml:/clab.yaml \
+		$(CLAB_IMAGE) clab destroy -t /clab.yaml
+	@$(MAKE) kind-clean
+
 .PHONY: uninstall
 uninstall:
 	bash dist/images/cleanup.sh
@@ -901,7 +941,8 @@ ipam-bench:
 .PHONY: clean
 clean:
 	$(RM) dist/images/kube-ovn dist/images/kube-ovn-cmd
-	$(RM) yamls/kind.yaml yamls/clab-bgp.yaml
+	$(RM) yamls/kind.yaml
+	$(RM) yamls/clab-bgp.yaml yamls/clab-bgp-ha.yaml
 	$(RM) ovn.yaml kube-ovn.yaml kube-ovn-crd.yaml
 	$(RM) ovn-ic-0.yaml ovn-ic-1.yaml
 	$(RM) kustomization.yaml kwok.yaml kwok-node.yaml

diff --git a/pkg/speaker/config.go b/pkg/speaker/config.go
@@ -40,8 +40,8 @@ type Configuration struct {
 	GrpcPort                    uint32
 	ClusterAs                   uint32
 	RouterID                    string
-	NeighborAddress             string
-	NeighborIPv6Address         string
+	NeighborAddresses           []string
+	NeighborIPv6Addresses       []string
 	NeighborAs                  uint32
 	AuthPassword                string
 	HoldTime                    float64
@@ -71,8 +71,8 @@ func ParseFlags() (*Configuration, error) {
 		argGrpcPort                    = pflag.Uint32("grpc-port", DefaultBGPGrpcPort, "The port for grpc to listen, default:50051")
 		argClusterAs                   = pflag.Uint32("cluster-as", DefaultBGPClusterAs, "The as number of container network, default 65000")
 		argRouterID                    = pflag.String("router-id", "", "The address for the speaker to use as router id, default the node ip")
-		argNeighborAddress             = pflag.String("neighbor-address", "", "The router address the speaker connects to.")
-		argNeighborIPv6Address         = pflag.String("neighbor-ipv6-address", "", "The router address the speaker connects to.")
+		argNeighborAddress             = pflag.String("neighbor-address", "", "Comma separated IPv4 router addresses the speaker connects to.")
+		argNeighborIPv6Address         = pflag.String("neighbor-ipv6-address", "", "Comma separated IPv6 router addresses the speaker connects to.")
 		argNeighborAs                  = pflag.Uint32("neighbor-as", DefaultBGPNeighborAs, "The router as number, default 65001")
 		argAuthPassword                = pflag.String("auth-password", "", "bgp peer auth password")
 		argHoldTime                    = pflag.Duration("holdtime", DefaultBGPHoldtime, "ovn-speaker goes down abnormally, the local saving time of BGP route will be affected.Holdtime must be in the range 3s to 65536s. (default 90s)")
@@ -107,12 +107,6 @@ func ParseFlags() (*Configuration, error) {
 	if *argRouterID != "" && net.ParseIP(*argRouterID) == nil {
 		return nil, fmt.Errorf("invalid router-id format: %s", *argRouterID)
 	}
-	if *argNeighborAddress != "" && net.ParseIP(*argNeighborAddress).To4() == nil {
-		return nil, fmt.Errorf("invalid neighbor-address format: %s", *argNeighborAddress)
-	}
-	if *argNeighborIPv6Address != "" && net.ParseIP(*argNeighborIPv6Address).To16() == nil {
-		return nil, fmt.Errorf("invalid neighbor-ipv6-address format: %s", *argNeighborIPv6Address)
-	}
 	if *argEbgpMultihopTTL < 1 || *argEbgpMultihopTTL > 255 {
 		return nil, errors.New("the bgp MultihopTtl must be in the range 1 to 255")
 	}
@@ -123,8 +117,6 @@ func ParseFlags() (*Configuration, error) {
 		GrpcPort:                    *argGrpcPort,
 		ClusterAs:                   *argClusterAs,
 		RouterID:                    *argRouterID,
-		NeighborAddress:             *argNeighborAddress,
-		NeighborIPv6Address:         *argNeighborIPv6Address,
 		NeighborAs:                  *argNeighborAs,
 		AuthPassword:                *argAuthPassword,
 		HoldTime:                    ht,
@@ -138,6 +130,23 @@ func ParseFlags() (*Configuration, error) {
 		EbgpMultihopTTL:             *argEbgpMultihopTTL,
 	}
 
+	if *argNeighborAddress != "" {
+		config.NeighborAddresses = strings.Split(*argNeighborAddress, ",")
+		for _, addr := range config.NeighborAddresses {
+			if ip := net.ParseIP(addr); ip == nil || ip.To4() == nil {
+				return nil, fmt.Errorf("invalid neighbor-address format: %s", *argNeighborAddress)
+			}
+		}
+	}
+	if *argNeighborIPv6Address != "" {
+		config.NeighborIPv6Addresses = strings.Split(*argNeighborIPv6Address, ",")
+		for _, addr := range config.NeighborIPv6Addresses {
+			if ip := net.ParseIP(addr); ip == nil || ip.To16() == nil {
+				return nil, fmt.Errorf("invalid neighbor-ipv6-address format: %s", *argNeighborIPv6Address)
+			}
+		}
+	}
+
 	if config.RouterID == "" {
 		config.RouterID = os.Getenv("POD_IP")
 		if config.RouterID == "" {
@@ -207,19 +216,16 @@ func (config *Configuration) checkGracefulRestartOptions() error {
 
 func (config *Configuration) initBgpServer() error {
 	maxSize := 256 << 20
-	peersMap := make(map[api.Family_Afi]string)
 	var listenPort int32 = -1
 	grpcOpts := []grpc.ServerOption{grpc.MaxRecvMsgSize(maxSize), grpc.MaxSendMsgSize(maxSize)}
 	s := gobgp.NewBgpServer(
 		gobgp.GrpcListenAddress(fmt.Sprintf("%s:%d", config.GrpcHost, config.GrpcPort)),
 		gobgp.GrpcOption(grpcOpts))
 	go s.Serve()
 
-	if config.NeighborAddress != "" {
-		peersMap[api.Family_AFI_IP] = config.NeighborAddress
-	}
-	if config.NeighborIPv6Address != "" {
-		peersMap[api.Family_AFI_IP6] = config.NeighborIPv6Address
+	peersMap := map[api.Family_Afi][]string{
+		api.Family_AFI_IP:  config.NeighborAddresses,
+		api.Family_AFI_IP6: config.NeighborIPv6Addresses,
 	}
 
 	if config.PassiveMode {
@@ -235,56 +241,57 @@ func (config *Configuration) initBgpServer() error {
 	}); err != nil {
 		return err
 	}
-	for ipFamily, address := range peersMap {
-		peer := &api.Peer{
-			Timers: &api.Timers{Config: &api.TimersConfig{HoldTime: uint64(config.HoldTime)}},
-			Conf: &api.PeerConf{
-				NeighborAddress: address,
-				PeerAsn:         config.NeighborAs,
-			},
-			Transport: &api.Transport{
-				PassiveMode: config.PassiveMode,
-			},
-		}
-		if config.EbgpMultihopTTL != DefaultEbgpMultiHop {
-			peer.EbgpMultihop = &api.EbgpMultihop{
-				Enabled:     true,
-				MultihopTtl: uint32(config.EbgpMultihopTTL),
+	for ipFamily, addresses := range peersMap {
+		for _, addr := range addresses {
+			peer := &api.Peer{
+				Timers: &api.Timers{Config: &api.TimersConfig{HoldTime: uint64(config.HoldTime)}},
+				Conf: &api.PeerConf{
+					NeighborAddress: addr,
+					PeerAsn:         config.NeighborAs,
+				},
+				Transport: &api.Transport{
+					PassiveMode: config.PassiveMode,
+				},
 			}
-		}
-		if config.AuthPassword != "" {
-			peer.Conf.AuthPassword = config.AuthPassword
-		}
-		if config.GracefulRestart {
-
-			if err := config.checkGracefulRestartOptions(); err != nil {
-				return err
+			if config.EbgpMultihopTTL != DefaultEbgpMultiHop {
+				peer.EbgpMultihop = &api.EbgpMultihop{
+					Enabled:     true,
+					MultihopTtl: uint32(config.EbgpMultihopTTL),
+				}
 			}
-			peer.GracefulRestart = &api.GracefulRestart{
-				Enabled:         true,
-				RestartTime:     uint32(config.GracefulRestartTime.Seconds()),
-				DeferralTime:    uint32(config.GracefulRestartDeferralTime.Seconds()),
-				LocalRestarting: true,
+			if config.AuthPassword != "" {
+				peer.Conf.AuthPassword = config.AuthPassword
 			}
-			peer.AfiSafis = []*api.AfiSafi{
-				{
-					Config: &api.AfiSafiConfig{
-						Family:  &api.Family{Afi: ipFamily, Safi: api.Family_SAFI_UNICAST},
-						Enabled: true,
-					},
-					MpGracefulRestart: &api.MpGracefulRestart{
-						Config: &api.MpGracefulRestartConfig{
+			if config.GracefulRestart {
+				if err := config.checkGracefulRestartOptions(); err != nil {
+					return err
+				}
+				peer.GracefulRestart = &api.GracefulRestart{
+					Enabled:         true,
+					RestartTime:     uint32(config.GracefulRestartTime.Seconds()),
+					DeferralTime:    uint32(config.GracefulRestartDeferralTime.Seconds()),
+					LocalRestarting: true,
+				}
+				peer.AfiSafis = []*api.AfiSafi{
+					{
+						Config: &api.AfiSafiConfig{
+							Family:  &api.Family{Afi: ipFamily, Safi: api.Family_SAFI_UNICAST},
 							Enabled: true,
 						},
+						MpGracefulRestart: &api.MpGracefulRestart{
+							Config: &api.MpGracefulRestartConfig{
+								Enabled: true,
+							},
+						},
 					},
-				},
+				}
 			}
-		}
 
-		if err := s.AddPeer(context.Background(), &api.AddPeerRequest{
-			Peer: peer,
-		}); err != nil {
-			return err
+			if err := s.AddPeer(context.Background(), &api.AddPeerRequest{
+				Peer: peer,
+			}); err != nil {
+				return err
+			}
 		}
 	}
 

diff --git a/pkg/speaker/subnet.go b/pkg/speaker/subnet.go
@@ -160,7 +160,7 @@ func (c *Controller) syncSubnetRoutes() {
 		}
 	}
 
-	if c.config.NeighborAddress != "" {
+	if len(c.config.NeighborAddresses) != 0 {
 		listPathRequest := &bgpapi.ListPathRequest{
 			TableType: bgpapi.TableType_GLOBAL,
 			Family:    &bgpapi.Family{Afi: bgpapi.Family_AFI_IP, Safi: bgpapi.Family_SAFI_UNICAST},
@@ -187,8 +187,7 @@ func (c *Controller) syncSubnetRoutes() {
 		}
 	}
 
-	if c.config.NeighborIPv6Address != "" {
-
+	if len(c.config.NeighborIPv6Addresses) != 0 {
 		listIPv6PathRequest := &bgpapi.ListPathRequest{
 			TableType: bgpapi.TableType_GLOBAL,
 			Family:    &bgpapi.Family{Afi: bgpapi.Family_AFI_IP6, Safi: bgpapi.Family_SAFI_UNICAST},
@@ -266,24 +265,26 @@ func (c *Controller) addRoute(route string) error {
 	if err != nil {
 		return err
 	}
-	_, err = c.config.BgpServer.AddPath(context.Background(), &bgpapi.AddPathRequest{
-		Path: &bgpapi.Path{
-			Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST},
-			Nlri:   nlri,
-			Pattrs: attrs,
-		},
-	})
-	if err != nil {
-		klog.Errorf("add path failed, %v", err)
-		return err
+	for _, attr := range attrs {
+		_, err = c.config.BgpServer.AddPath(context.Background(), &bgpapi.AddPathRequest{
+			Path: &bgpapi.Path{
+				Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST},
+				Nlri:   nlri,
+				Pattrs: attr,
+			},
+		})
+		if err != nil {
+			klog.Errorf("add path failed, %v", err)
+			return err
+		}
 	}
 	return nil
 }
 
-func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, []*anypb.Any, error) {
-	neighborAddr := c.config.NeighborAddress
+func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, [][]*anypb.Any, error) {
+	neighborAddresses := c.config.NeighborAddresses
 	if util.CheckProtocol(route) == kubeovnv1.ProtocolIPv6 {
-		neighborAddr = c.config.NeighborIPv6Address
+		neighborAddresses = c.config.NeighborIPv6Addresses
 	}
 
 	prefix, prefixLen, err := parseRoute(route)
@@ -294,13 +295,18 @@ func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, []*anypb.Any, er
 		Prefix:    prefix,
 		PrefixLen: prefixLen,
 	})
-	a1, _ := anypb.New(&bgpapi.OriginAttribute{
-		Origin: 0,
-	})
-	a2, _ := anypb.New(&bgpapi.NextHopAttribute{
-		NextHop: getNextHopAttribute(neighborAddr, c.config.RouterID),
-	})
-	attrs := []*anypb.Any{a1, a2}
+
+	attrs := make([][]*anypb.Any, 0, len(neighborAddresses))
+	for _, addr := range neighborAddresses {
+		a1, _ := anypb.New(&bgpapi.OriginAttribute{
+			Origin: 0,
+		})
+		a2, _ := anypb.New(&bgpapi.NextHopAttribute{
+			NextHop: getNextHopAttribute(addr, c.config.RouterID),
+		})
+		attrs = append(attrs, []*anypb.Any{a1, a2})
+	}
+
 	return nlri, attrs, err
 }
 
@@ -314,16 +320,18 @@ func (c *Controller) delRoute(route string) error {
 	if err != nil {
 		return err
 	}
-	err = c.config.BgpServer.DeletePath(context.Background(), &bgpapi.DeletePathRequest{
-		Path: &bgpapi.Path{
-			Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST},
-			Nlri:   nlri,
-			Pattrs: attrs,
-		},
-	})
-	if err != nil {
-		klog.Errorf("del path failed, %v", err)
-		return err
+	for _, attr := range attrs {
+		err = c.config.BgpServer.DeletePath(context.Background(), &bgpapi.DeletePathRequest{
+			Path: &bgpapi.Path{
+				Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST},
+				Nlri:   nlri,
+				Pattrs: attr,
+			},
+		})
+		if err != nil {
+			klog.Errorf("del path failed, %v", err)
+			return err
+		}
 	}
 	return nil
 }