diff --git a/.gitignore b/.gitignore index 3ec002f1f3b..b2ef3ccdf84 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ broker-info.subm broker-info.subm.* broker-info-internal.subm yamls/clab-bgp.yaml +yamls/clab-bgp-ha.yaml kube-ovn.tar vpc-nat-gateway.tar image-amd64.tar diff --git a/Makefile b/Makefile index 714ae6a8221..8111e10f6ab 100644 --- a/Makefile +++ b/Makefile @@ -416,6 +416,19 @@ kind-init-bgp: kind-clean-bgp kind-init -v $(CURDIR)/yamls/clab-bgp.yaml:/clab.yaml \ $(CLAB_IMAGE) clab deploy -t /clab.yaml +.PHONY: kind-init-bgp-ha +kind-init-bgp-ha: kind-clean-bgp kind-init + kube_ovn_version=$(VERSION) j2 yamls/clab-bgp-ha.yaml.j2 -o yamls/clab-bgp-ha.yaml + docker run --rm --privileged \ + --name kube-ovn-bgp \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/netns:/var/run/netns \ + -v /var/lib/docker/containers:/var/lib/docker/containers \ + --pid=host \ + -v $(CURDIR)/yamls/clab-bgp-ha.yaml:/clab.yaml \ + $(CLAB_IMAGE) clab deploy -t /clab.yaml + .PHONY: kind-load-image kind-load-image: $(call kind_load_image,kube-ovn,$(REGISTRY)/kube-ovn:$(VERSION)) @@ -792,6 +805,20 @@ kind-install-bgp: kind-install -e 's/--cluster-as=.*/--cluster-as=65002/' yamls/speaker.yaml | \ kubectl apply -f - kubectl -n kube-system rollout status ds kube-ovn-speaker --timeout 60s + docker exec clab-bgp-router vtysh -c "show ip route bgp" + +.PHONY: kind-install-bgp-ha +kind-install-bgp-ha: kind-install + kubectl label node --all ovn.kubernetes.io/bgp=true + kubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=local + sed -e 's#image: .*#image: $(REGISTRY)/kube-ovn:$(VERSION)#' \ + -e 's/--neighbor-address=.*/--neighbor-address=10.0.1.1,10.0.1.2/' \ + -e 's/--neighbor-as=.*/--neighbor-as=65001/' \ + -e 's/--cluster-as=.*/--cluster-as=65002/' yamls/speaker.yaml | \ + kubectl apply -f - + kubectl -n kube-system rollout status ds kube-ovn-speaker --timeout 60s + docker exec clab-bgp-router-1 vtysh -c "show ip route bgp" + docker exec clab-bgp-router-2 vtysh -c "show ip route bgp" .PHONY: kind-install-deepflow kind-install-deepflow: kind-install @@ -845,8 +872,7 @@ kind-clean-ovn-submariner: kind-clean kind delete cluster --name=kube-ovn1 .PHONY: kind-clean-bgp -kind-clean-bgp: - $(call docker_rm_container,kube-ovn-bgp) +kind-clean-bgp: kind-clean-bgp-ha kube_ovn_version=$(VERSION) j2 yamls/clab-bgp.yaml.j2 -o yamls/clab-bgp.yaml docker run --rm --privileged \ --name kube-ovn-bgp \ @@ -859,6 +885,20 @@ kind-clean-bgp: $(CLAB_IMAGE) clab destroy -t /clab.yaml @$(MAKE) kind-clean +.PHONY: kind-clean-bgp-ha +kind-clean-bgp-ha: + kube_ovn_version=$(VERSION) j2 yamls/clab-bgp-ha.yaml.j2 -o yamls/clab-bgp-ha.yaml + docker run --rm --privileged \ + --name kube-ovn-bgp \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/netns:/var/run/netns \ + -v /var/lib/docker/containers:/var/lib/docker/containers \ + --pid=host \ + -v $(CURDIR)/yamls/clab-bgp-ha.yaml:/clab.yaml \ + $(CLAB_IMAGE) clab destroy -t /clab.yaml + @$(MAKE) kind-clean + .PHONY: uninstall uninstall: bash dist/images/cleanup.sh @@ -901,7 +941,8 @@ ipam-bench: .PHONY: clean clean: $(RM) dist/images/kube-ovn dist/images/kube-ovn-cmd - $(RM) yamls/kind.yaml yamls/clab-bgp.yaml + $(RM) yamls/kind.yaml + $(RM) yamls/clab-bgp.yaml yamls/clab-bgp-ha.yaml $(RM) ovn.yaml kube-ovn.yaml kube-ovn-crd.yaml $(RM) ovn-ic-0.yaml ovn-ic-1.yaml $(RM) kustomization.yaml kwok.yaml kwok-node.yaml diff --git a/pkg/speaker/config.go b/pkg/speaker/config.go index f3b3a018ef8..9cd6db02a00 100644 --- a/pkg/speaker/config.go +++ b/pkg/speaker/config.go @@ -40,8 +40,8 @@ type Configuration struct { GrpcPort uint32 ClusterAs uint32 RouterID string - NeighborAddress string - NeighborIPv6Address string + NeighborAddresses []string + NeighborIPv6Addresses []string NeighborAs uint32 AuthPassword string HoldTime float64 @@ -71,8 +71,8 @@ func ParseFlags() (*Configuration, error) { argGrpcPort = pflag.Uint32("grpc-port", DefaultBGPGrpcPort, "The port for grpc to listen, default:50051") argClusterAs = pflag.Uint32("cluster-as", DefaultBGPClusterAs, "The as number of container network, default 65000") argRouterID = pflag.String("router-id", "", "The address for the speaker to use as router id, default the node ip") - argNeighborAddress = pflag.String("neighbor-address", "", "The router address the speaker connects to.") - argNeighborIPv6Address = pflag.String("neighbor-ipv6-address", "", "The router address the speaker connects to.") + argNeighborAddress = pflag.String("neighbor-address", "", "Comma separated IPv4 router addresses the speaker connects to.") + argNeighborIPv6Address = pflag.String("neighbor-ipv6-address", "", "Comma separated IPv6 router addresses the speaker connects to.") argNeighborAs = pflag.Uint32("neighbor-as", DefaultBGPNeighborAs, "The router as number, default 65001") argAuthPassword = pflag.String("auth-password", "", "bgp peer auth password") argHoldTime = pflag.Duration("holdtime", DefaultBGPHoldtime, "ovn-speaker goes down abnormally, the local saving time of BGP route will be affected.Holdtime must be in the range 3s to 65536s. (default 90s)") @@ -107,12 +107,6 @@ func ParseFlags() (*Configuration, error) { if *argRouterID != "" && net.ParseIP(*argRouterID) == nil { return nil, fmt.Errorf("invalid router-id format: %s", *argRouterID) } - if *argNeighborAddress != "" && net.ParseIP(*argNeighborAddress).To4() == nil { - return nil, fmt.Errorf("invalid neighbor-address format: %s", *argNeighborAddress) - } - if *argNeighborIPv6Address != "" && net.ParseIP(*argNeighborIPv6Address).To16() == nil { - return nil, fmt.Errorf("invalid neighbor-ipv6-address format: %s", *argNeighborIPv6Address) - } if *argEbgpMultihopTTL < 1 || *argEbgpMultihopTTL > 255 { return nil, errors.New("the bgp MultihopTtl must be in the range 1 to 255") } @@ -123,8 +117,6 @@ func ParseFlags() (*Configuration, error) { GrpcPort: *argGrpcPort, ClusterAs: *argClusterAs, RouterID: *argRouterID, - NeighborAddress: *argNeighborAddress, - NeighborIPv6Address: *argNeighborIPv6Address, NeighborAs: *argNeighborAs, AuthPassword: *argAuthPassword, HoldTime: ht, @@ -138,6 +130,23 @@ func ParseFlags() (*Configuration, error) { EbgpMultihopTTL: *argEbgpMultihopTTL, } + if *argNeighborAddress != "" { + config.NeighborAddresses = strings.Split(*argNeighborAddress, ",") + for _, addr := range config.NeighborAddresses { + if ip := net.ParseIP(addr); ip == nil || ip.To4() == nil { + return nil, fmt.Errorf("invalid neighbor-address format: %s", *argNeighborAddress) + } + } + } + if *argNeighborIPv6Address != "" { + config.NeighborIPv6Addresses = strings.Split(*argNeighborIPv6Address, ",") + for _, addr := range config.NeighborIPv6Addresses { + if ip := net.ParseIP(addr); ip == nil || ip.To16() == nil { + return nil, fmt.Errorf("invalid neighbor-ipv6-address format: %s", *argNeighborIPv6Address) + } + } + } + if config.RouterID == "" { config.RouterID = os.Getenv("POD_IP") if config.RouterID == "" { @@ -207,7 +216,6 @@ func (config *Configuration) checkGracefulRestartOptions() error { func (config *Configuration) initBgpServer() error { maxSize := 256 << 20 - peersMap := make(map[api.Family_Afi]string) var listenPort int32 = -1 grpcOpts := []grpc.ServerOption{grpc.MaxRecvMsgSize(maxSize), grpc.MaxSendMsgSize(maxSize)} s := gobgp.NewBgpServer( @@ -215,11 +223,9 @@ func (config *Configuration) initBgpServer() error { gobgp.GrpcOption(grpcOpts)) go s.Serve() - if config.NeighborAddress != "" { - peersMap[api.Family_AFI_IP] = config.NeighborAddress - } - if config.NeighborIPv6Address != "" { - peersMap[api.Family_AFI_IP6] = config.NeighborIPv6Address + peersMap := map[api.Family_Afi][]string{ + api.Family_AFI_IP: config.NeighborAddresses, + api.Family_AFI_IP6: config.NeighborIPv6Addresses, } if config.PassiveMode { @@ -235,56 +241,57 @@ func (config *Configuration) initBgpServer() error { }); err != nil { return err } - for ipFamily, address := range peersMap { - peer := &api.Peer{ - Timers: &api.Timers{Config: &api.TimersConfig{HoldTime: uint64(config.HoldTime)}}, - Conf: &api.PeerConf{ - NeighborAddress: address, - PeerAsn: config.NeighborAs, - }, - Transport: &api.Transport{ - PassiveMode: config.PassiveMode, - }, - } - if config.EbgpMultihopTTL != DefaultEbgpMultiHop { - peer.EbgpMultihop = &api.EbgpMultihop{ - Enabled: true, - MultihopTtl: uint32(config.EbgpMultihopTTL), + for ipFamily, addresses := range peersMap { + for _, addr := range addresses { + peer := &api.Peer{ + Timers: &api.Timers{Config: &api.TimersConfig{HoldTime: uint64(config.HoldTime)}}, + Conf: &api.PeerConf{ + NeighborAddress: addr, + PeerAsn: config.NeighborAs, + }, + Transport: &api.Transport{ + PassiveMode: config.PassiveMode, + }, } - } - if config.AuthPassword != "" { - peer.Conf.AuthPassword = config.AuthPassword - } - if config.GracefulRestart { - - if err := config.checkGracefulRestartOptions(); err != nil { - return err + if config.EbgpMultihopTTL != DefaultEbgpMultiHop { + peer.EbgpMultihop = &api.EbgpMultihop{ + Enabled: true, + MultihopTtl: uint32(config.EbgpMultihopTTL), + } } - peer.GracefulRestart = &api.GracefulRestart{ - Enabled: true, - RestartTime: uint32(config.GracefulRestartTime.Seconds()), - DeferralTime: uint32(config.GracefulRestartDeferralTime.Seconds()), - LocalRestarting: true, + if config.AuthPassword != "" { + peer.Conf.AuthPassword = config.AuthPassword } - peer.AfiSafis = []*api.AfiSafi{ - { - Config: &api.AfiSafiConfig{ - Family: &api.Family{Afi: ipFamily, Safi: api.Family_SAFI_UNICAST}, - Enabled: true, - }, - MpGracefulRestart: &api.MpGracefulRestart{ - Config: &api.MpGracefulRestartConfig{ + if config.GracefulRestart { + if err := config.checkGracefulRestartOptions(); err != nil { + return err + } + peer.GracefulRestart = &api.GracefulRestart{ + Enabled: true, + RestartTime: uint32(config.GracefulRestartTime.Seconds()), + DeferralTime: uint32(config.GracefulRestartDeferralTime.Seconds()), + LocalRestarting: true, + } + peer.AfiSafis = []*api.AfiSafi{ + { + Config: &api.AfiSafiConfig{ + Family: &api.Family{Afi: ipFamily, Safi: api.Family_SAFI_UNICAST}, Enabled: true, }, + MpGracefulRestart: &api.MpGracefulRestart{ + Config: &api.MpGracefulRestartConfig{ + Enabled: true, + }, + }, }, - }, + } } - } - if err := s.AddPeer(context.Background(), &api.AddPeerRequest{ - Peer: peer, - }); err != nil { - return err + if err := s.AddPeer(context.Background(), &api.AddPeerRequest{ + Peer: peer, + }); err != nil { + return err + } } } diff --git a/pkg/speaker/subnet.go b/pkg/speaker/subnet.go index d2fbd7c9740..23ec2522f7c 100644 --- a/pkg/speaker/subnet.go +++ b/pkg/speaker/subnet.go @@ -160,7 +160,7 @@ func (c *Controller) syncSubnetRoutes() { } } - if c.config.NeighborAddress != "" { + if len(c.config.NeighborAddresses) != 0 { listPathRequest := &bgpapi.ListPathRequest{ TableType: bgpapi.TableType_GLOBAL, Family: &bgpapi.Family{Afi: bgpapi.Family_AFI_IP, Safi: bgpapi.Family_SAFI_UNICAST}, @@ -187,8 +187,7 @@ func (c *Controller) syncSubnetRoutes() { } } - if c.config.NeighborIPv6Address != "" { - + if len(c.config.NeighborIPv6Addresses) != 0 { listIPv6PathRequest := &bgpapi.ListPathRequest{ TableType: bgpapi.TableType_GLOBAL, Family: &bgpapi.Family{Afi: bgpapi.Family_AFI_IP6, Safi: bgpapi.Family_SAFI_UNICAST}, @@ -266,24 +265,26 @@ func (c *Controller) addRoute(route string) error { if err != nil { return err } - _, err = c.config.BgpServer.AddPath(context.Background(), &bgpapi.AddPathRequest{ - Path: &bgpapi.Path{ - Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, - Nlri: nlri, - Pattrs: attrs, - }, - }) - if err != nil { - klog.Errorf("add path failed, %v", err) - return err + for _, attr := range attrs { + _, err = c.config.BgpServer.AddPath(context.Background(), &bgpapi.AddPathRequest{ + Path: &bgpapi.Path{ + Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, + Nlri: nlri, + Pattrs: attr, + }, + }) + if err != nil { + klog.Errorf("add path failed, %v", err) + return err + } } return nil } -func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, []*anypb.Any, error) { - neighborAddr := c.config.NeighborAddress +func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, [][]*anypb.Any, error) { + neighborAddresses := c.config.NeighborAddresses if util.CheckProtocol(route) == kubeovnv1.ProtocolIPv6 { - neighborAddr = c.config.NeighborIPv6Address + neighborAddresses = c.config.NeighborIPv6Addresses } prefix, prefixLen, err := parseRoute(route) @@ -294,13 +295,18 @@ func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, []*anypb.Any, er Prefix: prefix, PrefixLen: prefixLen, }) - a1, _ := anypb.New(&bgpapi.OriginAttribute{ - Origin: 0, - }) - a2, _ := anypb.New(&bgpapi.NextHopAttribute{ - NextHop: getNextHopAttribute(neighborAddr, c.config.RouterID), - }) - attrs := []*anypb.Any{a1, a2} + + attrs := make([][]*anypb.Any, 0, len(neighborAddresses)) + for _, addr := range neighborAddresses { + a1, _ := anypb.New(&bgpapi.OriginAttribute{ + Origin: 0, + }) + a2, _ := anypb.New(&bgpapi.NextHopAttribute{ + NextHop: getNextHopAttribute(addr, c.config.RouterID), + }) + attrs = append(attrs, []*anypb.Any{a1, a2}) + } + return nlri, attrs, err } @@ -314,16 +320,18 @@ func (c *Controller) delRoute(route string) error { if err != nil { return err } - err = c.config.BgpServer.DeletePath(context.Background(), &bgpapi.DeletePathRequest{ - Path: &bgpapi.Path{ - Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, - Nlri: nlri, - Pattrs: attrs, - }, - }) - if err != nil { - klog.Errorf("del path failed, %v", err) - return err + for _, attr := range attrs { + err = c.config.BgpServer.DeletePath(context.Background(), &bgpapi.DeletePathRequest{ + Path: &bgpapi.Path{ + Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, + Nlri: nlri, + Pattrs: attr, + }, + }) + if err != nil { + klog.Errorf("del path failed, %v", err) + return err + } } return nil } diff --git a/yamls/clab-bgp-ha.yaml.j2 b/yamls/clab-bgp-ha.yaml.j2 new file mode 100644 index 00000000000..f225fc9919e --- /dev/null +++ b/yamls/clab-bgp-ha.yaml.j2 @@ -0,0 +1,98 @@ +name: bgp +topology: + kinds: + linux: + image: kubeovn/kube-ovn:{{ kube_ovn_version }} + cmd: bash + + nodes: + switch: + kind: linux + exec: + - ip link add br0 type bridge + - ip link set net1 master br0 + - ip link set net2 master br0 + - ip link set net3 master br0 + - ip link set net4 master br0 + - ip link set net5 master br0 + - ip link set net6 master br0 + - ip link set net7 master br0 + - ip link set br0 up + router-1: + kind: linux + image: frrouting/frr:v8.4.1 + labels: + app: frr + exec: + - ip link delete eth0 + - ip address add 10.0.1.1/24 dev net1 + - ip address add 10.0.2.1/24 dev net2 + - touch /etc/frr/vtysh.conf + - sed -i -e 's/bgpd=no/bgpd=yes/g' /etc/frr/daemons + - /usr/lib/frr/frrinit.sh start + - >- + vtysh -c 'conf t' + -c 'frr defaults datacenter' + -c 'router bgp 65001' + -c ' bgp router-id 10.0.1.1' + -c ' no bgp ebgp-requires-policy' + -c ' neighbor SERVERS peer-group' + -c ' neighbor SERVERS remote-as external' + -c ' neighbor 10.0.1.101 peer-group SERVERS' + -c ' neighbor 10.0.1.102 peer-group SERVERS' + -c ' address-family ipv4 unicast' + -c ' redistribute connected' + -c ' exit-address-family' + -c '!' + router-2: + kind: linux + image: frrouting/frr:v8.4.1 + labels: + app: frr + exec: + - ip link delete eth0 + - ip address add 10.0.1.2/24 dev net1 + - ip address add 10.0.2.2/24 dev net2 + - touch /etc/frr/vtysh.conf + - sed -i -e 's/bgpd=no/bgpd=yes/g' /etc/frr/daemons + - /usr/lib/frr/frrinit.sh start + - >- + vtysh -c 'conf t' + -c 'frr defaults datacenter' + -c 'router bgp 65001' + -c ' bgp router-id 10.0.1.2' + -c ' no bgp ebgp-requires-policy' + -c ' neighbor SERVERS peer-group' + -c ' neighbor SERVERS remote-as external' + -c ' neighbor 10.0.1.101 peer-group SERVERS' + -c ' neighbor 10.0.1.102 peer-group SERVERS' + -c ' address-family ipv4 unicast' + -c ' redistribute connected' + -c ' exit-address-family' + -c '!' + k8s-master: + kind: linux + network-mode: container:kube-ovn-control-plane + exec: + - ip address add 10.0.1.101/24 dev net1 + - ip route add 10.0.0.0/16 via 10.0.1.1 + k8s-worker: + kind: linux + network-mode: container:kube-ovn-worker + exec: + - ip address add 10.0.1.102/24 dev net1 + - ip route add 10.0.0.0/16 via 10.0.1.1 + ext: + kind: linux + exec: + - ip address add 10.0.2.101/24 dev net1 + - ip route replace default nexthop via 10.0.2.1 weight 1 nexthop via 10.0.2.2 weight 1 + + links: + - endpoints: ["switch:net1", "router-1:net1"] + - endpoints: ["switch:net2", "router-1:net2"] + - endpoints: ["switch:net3", "router-2:net1"] + - endpoints: ["switch:net4", "router-2:net2"] + - endpoints: ["switch:net5", "k8s-master:net1"] + - endpoints: ["switch:net6", "k8s-worker:net1"] + - endpoints: ["switch:net7", "ext:net1"]