ovn-ic-ecmp refactor

Makefile

@@ -359,19 +359,19 @@ kind-init-ovn-ic: kind-init-ovn-ic-ipv4
 
 .PHONY: kind-init-ovn-ic-ipv4
 kind-init-ovn-ic-ipv4: kind-clean-ovn-ic
-	@ovn_ic=true $(MAKE) kind-init
+	@ha=true $(MAKE) kind-init
 	@ovn_ic=true $(MAKE) kind-generate-config
 	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)
 
 .PHONY: kind-init-ovn-ic-ipv6
 kind-init-ovn-ic-ipv6: kind-clean-ovn-ic
-	@ovn_ic=true $(MAKE) kind-init-ipv6
+	@ha=true $(MAKE) kind-init-ipv6
 	@ovn_ic=true ip_family=ipv6 $(MAKE) kind-generate-config
 	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)
 
 .PHONY: kind-init-ovn-ic-dual
 kind-init-ovn-ic-dual: kind-clean-ovn-ic
-	@ovn_ic=true $(MAKE) kind-init-dual
+	@ha=true $(MAKE) kind-init-dual
 	@ovn_ic=true ip_family=dual $(MAKE) kind-generate-config
 	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)
 
@@ -467,6 +467,7 @@ kind-install-chart: kind-load-image kind-untaint-control-plane
 		--set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \
 		--set networking.ENABLE_SSL=$(shell echo $${ENABLE_SSL:-false}) \
 		--set func.ENABLE_BIND_LOCAL_IP=$(shell echo $${ENABLE_BIND_LOCAL_IP:-true})
+		--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
 	sleep 60
 	kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
 	kubectl -n kube-system rollout status --timeout=1s daemonset/ovs-ovn
@@ -482,6 +483,7 @@ kind-install-chart-ssl:
 kind-upgrade-chart: kind-load-image
 	helm upgrade kubeovn ./charts \
 		--set global.images.kubeovn.tag=$(VERSION)
+		--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
 	sleep 90
 	kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
 	kubectl -n kube-system wait pod --for=condition=ready -l app=ovs
@@ -522,31 +524,33 @@ kind-install-overlay-ipv4: kind-install
 kind-install-ovn-ic: kind-install-ovn-ic-ipv4
 
 .PHONY: kind-install-ovn-ic-ipv4
-kind-install-ovn-ic-ipv4: kind-install
+kind-install-ovn-ic-ipv4:
+	@ENABLE_IC=true $(MAKE) kind-install
 	$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
 	kubectl config use-context kind-kube-ovn1
 	@$(MAKE) kind-untaint-control-plane
 	sed -e 's/10.16.0/10.18.0/g' \
 		-e 's/10.96.0/10.98.0/g' \
 		-e 's/100.64.0/100.68.0/g' \
 		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
-		dist/images/install.sh | bash
+		dist/images/install.sh | ENABLE_IC=true bash
 	kubectl describe no
 
-	docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
-	@set -e; \
-	ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
-	zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
-	zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
 	kubectl config use-context kind-kube-ovn
+	sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash
+
+	@set -e; \
+	ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
+	ic_db_host=$${ic_db_host%?}; \
+	zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
+	zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
 	kubectl apply -f ovn-ic-0.yaml
 	kubectl config use-context kind-kube-ovn1
 	kubectl apply -f ovn-ic-1.yaml
-	sleep 6
-	docker exec ovn-ic-db ovn-ic-sbctl show
 
 .PHONY: kind-install-ovn-ic-ipv6
-kind-install-ovn-ic-ipv6: kind-install-ipv6
+kind-install-ovn-ic-ipv6:
+	@ENABLE_IC=true $(MAKE) kind-install-ipv6
 	$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
 	kubectl config use-context kind-kube-ovn1
 	@$(MAKE) kind-untaint-control-plane
@@ -555,23 +559,24 @@ kind-install-ovn-ic-ipv6: kind-install-ipv6
 		-e 's/fd00:100:64:/fd00:100:68:/g' \
 		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
 		dist/images/install.sh | \
-		IPV6=true bash
+		IPV6=true ENABLE_IC=true bash
 	kubectl describe no
 
-	docker run -d --name ovn-ic-db --network kind -e PROTOCOL="ipv6" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
-	@set -e; \
-	ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"); \
-	zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
-	zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
 	kubectl config use-context kind-kube-ovn
+	sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash
+
+	@set -e; \
+	ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
+	ic_db_host=$${ic_db_host%?}; \
+	zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
+	zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
 	kubectl apply -f ovn-ic-0.yaml
 	kubectl config use-context kind-kube-ovn1
 	kubectl apply -f ovn-ic-1.yaml
-	sleep 6
-	docker exec ovn-ic-db ovn-ic-sbctl show
 
 .PHONY: kind-install-ovn-ic-dual
-kind-install-ovn-ic-dual: kind-install-dual
+kind-install-ovn-ic-dual:
+	@ENABLE_IC=true $(MAKE) kind-install-dual
 	$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
 	kubectl config use-context kind-kube-ovn1
 	@$(MAKE) kind-untaint-control-plane
@@ -583,21 +588,20 @@ kind-install-ovn-ic-dual: kind-install-dual
 		-e 's/fd00:100:64:/fd00:100:68:/g' \
 		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
 		dist/images/install.sh | \
-		DUAL_STACK=true bash
+		DUAL_STACK=true ENABLE_IC=true bash
 	kubectl describe no
 
-	docker run -d --name ovn-ic-db --network kind -e PROTOCOL="dual" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
-	@set -e; \
-
-	ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
-	zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
-	zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
 	kubectl config use-context kind-kube-ovn
+	sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash
+
+	@set -e; \
+	ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
+	ic_db_host=$${ic_db_host%?}; \
+	zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
+	zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
 	kubectl apply -f ovn-ic-0.yaml
 	kubectl config use-context kind-kube-ovn1
 	kubectl apply -f ovn-ic-1.yaml
-	sleep 6
-	docker exec ovn-ic-db ovn-ic-sbctl show
 
 .PHONY: kind-install-ovn-submariner
 kind-install-ovn-submariner: kind-install

charts/templates/ic-controller-deploy.yaml

@@ -0,0 +1,109 @@
+{{- if eq .Values.func.ENABLE_IC true }}
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ovn-ic-controller
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      OVN IC Client
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: ovn-ic-controller
+  template:
+    metadata:
+      labels:
+        app: ovn-ic-controller
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: ovn-ic-controller
+              topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
+      serviceAccountName: ovn
+      hostNetwork: true
+      containers:
+        - name: ovn-ic-controller
+          image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["/kube-ovn/start-ic-controller.sh"]
+          args:
+          - --log_file=/var/log/kube-ovn/kube-ovn-ic-controller.log
+          - --log_file_max_size=0
+          - --logtostderr=false
+          - --alsologtostderr=true
+          securityContext:
+            capabilities:
+              add: ["SYS_NICE"]
+          env:
+            - name: ENABLE_SSL
+              value: "{{ .Values.networking.ENABLE_SSL }}"
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: OVN_DB_IPS
+              value: "{{ .Values.MASTER_NODES }}"
+          resources:
+            requests:
+              cpu: 300m
+              memory: 200Mi
+            limits:
+              cpu: 3
+              memory: 1Gi
+          volumeMounts:
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /etc/ovn
+              name: host-config-ovn
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+            - mountPath: /etc/localtime
+              name: localtime
+            - mountPath: /var/run/tls
+              name: kube-ovn-tls
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
+      nodeSelector:
+        kubernetes.io/os: "linux"
+        kube-ovn/role: "master"
+      volumes:
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: host-config-ovn
+          hostPath:
+            path: /etc/origin/ovn
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
+        - name: kube-ovn-tls
+          secret:
+            optional: true
+            secretName: kube-ovn-tls
+{{- end }}

cmd/cmdmain.go

@@ -14,6 +14,7 @@ import (
 	"github.com/kubeovn/kube-ovn/cmd/controller"
 	"github.com/kubeovn/kube-ovn/cmd/controller_health_check"
 	"github.com/kubeovn/kube-ovn/cmd/daemon"
+	"github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller"
 	"github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker"
 	"github.com/kubeovn/kube-ovn/cmd/ovn_monitor"
 	"github.com/kubeovn/kube-ovn/cmd/pinger"
@@ -29,6 +30,7 @@ const (
 	CmdSpeaker               = "kube-ovn-speaker"
 	CmdControllerHealthCheck = "kube-ovn-controller-healthcheck"
 	CmdOvnLeaderChecker      = "kube-ovn-leader-checker"
+	CmdOvnICController       = "kube-ovn-ic-controller"
 )
 
 const timeFormat = "2006-01-02_15:04:05"
@@ -112,6 +114,8 @@ func main() {
 		controller_health_check.CmdMain()
 	case CmdOvnLeaderChecker:
 		ovn_leader_checker.CmdMain()
+	case CmdOvnICController:
+		ovn_ic_controller.CmdMain()
 	default:
 		util.LogFatalAndExit(nil, "%s is an unknown command", cmd)
 	}

cmd/ovn_ic_controller/ovn_ic_controller.go

@@ -0,0 +1,24 @@
+package ovn_ic_controller
+
+import (
+	"k8s.io/klog/v2"
+	"k8s.io/sample-controller/pkg/signals"
+
+	"github.com/kubeovn/kube-ovn/pkg/ovn_ic_controller"
+	"github.com/kubeovn/kube-ovn/pkg/util"
+	"github.com/kubeovn/kube-ovn/versions"
+)
+
+func CmdMain() {
+	defer klog.Flush()
+
+	klog.Infof(versions.String())
+	config, err := ovn_ic_controller.ParseFlags()
+	if err != nil {
+		util.LogFatalAndExit(err, "failed to parse config")
+	}
+
+	stopCh := signals.SetupSignalHandler().Done()
+	ctl := ovn_ic_controller.NewController(config)
+	ctl.Run(stopCh)
+}

dist/images/Dockerfile

@@ -26,4 +26,5 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \
-    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller

dist/images/cleanup.sh

@@ -89,6 +89,8 @@ kubectl delete --ignore-not-found deploy kube-ovn-monitor -n kube-system
 kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-config -n kube-system
 kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system
 kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system
+kubectl delete --ignore-not-found deploy ovn-ic-controller -n kube-system
+kubectl delete --ignore-not-found deploy ovn-ic-server -n kube-system
 
 # wait for provier-networks to be deleted before deleting kube-ovn-cni
 sleep 5