Use Calico CNI for docker clusters in e2e

rancher · Mar 14, 2024 · cfbfb41 · cfbfb41
1 parent 7e54296
commit cfbfb41
Show file tree

Hide file tree

Showing 15 changed files with 238 additions and 146 deletions.
diff --git a/test/e2e/const.go b/test/e2e/const.go
@@ -74,6 +74,12 @@ var (
 
 	//go:embed data/cluster-templates/vsphere-kubeadm.yaml
 	CAPIvSphereKubeadm []byte
+
+	//go:embed data/cluster-templates/cni-calico.yaml
+	CalicoCNI []byte
+
+	//go:embed data/cluster-templates/cni-kindnet.yaml
+	KindnetCNI []byte
 )
 
 const (

diff --git a/test/e2e/data/capi-operator/capa-variables.yaml b/test/e2e/data/capi-operator/capa-variables.yaml
@@ -3,12 +3,16 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: capa-system
+  annotations:
+    "helm.sh/resource-policy": keep
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: full-variables
   namespace: capa-system
+  annotations:
+    "helm.sh/resource-policy": keep
 type: Opaque
 stringData:
   AWS_B64ENCODED_CREDENTIALS: "{{ .AWSEncodedCredentials }}"
diff --git a/test/e2e/data/capi-operator/capi-providers.yaml b/test/e2e/data/capi-operator/capi-providers.yaml
@@ -3,14 +3,30 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: capd-system
+  annotations:
+    "helm.sh/resource-policy": keep
 ---
 apiVersion: turtles-capi.cattle.io/v1alpha1
 kind: CAPIProvider
 metadata:
   name: docker
   namespace: capd-system
+  annotations:
+    "helm.sh/resource-policy": keep
 spec:
   name: docker
   type: infrastructure
   configSecret:
     name: variables
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: helm
+  namespace: capd-system
+  annotations:
+    "helm.sh/resource-policy": keep
+spec:
+  type: addon
+  name: helm
+
diff --git a/test/e2e/data/capi-operator/capv-provider.yaml b/test/e2e/data/capi-operator/capv-provider.yaml
@@ -4,6 +4,8 @@ kind: CAPIProvider
 metadata:
   name: vsphere
   namespace: capv-system
+  annotations:
+    "helm.sh/resource-policy": keep
 spec:
   name: vsphere
   type: infrastructure

diff --git a/test/e2e/data/capi-operator/capv-variables.yaml b/test/e2e/data/capi-operator/capv-variables.yaml
@@ -3,12 +3,16 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: capv-system
+  annotations:
+    "helm.sh/resource-policy": keep
 ---
 apiVersion: v1
 kind: Secret
 metadata:
   name: vsphere-variables
   namespace: capv-system
+  annotations:
+    "helm.sh/resource-policy": keep
 type: Opaque
 stringData:
   VSPHERE_USERNAME: "${VSPHERE_USERNAME}"

diff --git a/test/e2e/data/capi-operator/capz-identity-secret.yaml b/test/e2e/data/capi-operator/capz-identity-secret.yaml
@@ -3,6 +3,8 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: capz-system
+  annotations:
+    "helm.sh/resource-policy": keep
 ---
 apiVersion: v1
 stringData:
@@ -11,4 +13,6 @@ kind: Secret
 metadata:
   name: cluster-identity-secret
   namespace: capz-system
+  annotations:
+    "helm.sh/resource-policy": keep
 type: Opaque
diff --git a/test/e2e/data/capi-operator/full-providers.yaml b/test/e2e/data/capi-operator/full-providers.yaml
@@ -4,6 +4,8 @@ kind: CAPIProvider
 metadata:
   name: aws
   namespace: capa-system
+  annotations:
+    "helm.sh/resource-policy": keep
 spec:
   type: infrastructure
   name: aws
@@ -20,6 +22,8 @@ kind: CAPIProvider
 metadata:
   name: azure
   namespace: capz-system
+  annotations:
+    "helm.sh/resource-policy": keep
 spec:
   type: infrastructure
   name: azure

diff --git a/test/e2e/data/cluster-templates/cni-calico.yaml b/test/e2e/data/cluster-templates/cni-calico.yaml
@@ -0,0 +1,28 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: cluster-calico-cni
+  annotations:
+    "helm.sh/resource-policy": keep
+spec:
+  clusterSelector:
+    matchLabels:
+      calico: "true"
+  releaseName: calico
+  repoURL: https://docs.tigera.io/calico/charts
+  chartName: tigera-operator
+  namespace: kube-system
+  valuesTemplate: |
+    installation:
+      cni:
+        type: Calico
+        ipam:
+          type: HostLocal
+      calicoNetwork:
+        bgp: Disabled
+        mtu: 1350
+        ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
+        - cidr: {{ $cidr }}
+          encapsulation: None
+          natOutgoing: Enabled
+          nodeSelector: all(){{end}}
diff --git a/test/e2e/data/cluster-templates/cni-kindnet.yaml b/test/e2e/data/cluster-templates/cni-kindnet.yaml
@@ -0,0 +1,137 @@
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: ${CLUSTER_NAME}-crs-0
+  annotations:
+    "helm.sh/resource-policy": keep
+spec:
+  clusterSelector:
+    matchLabels:
+      cni: ${CLUSTER_NAME}-crs-0
+  resources:
+  - kind: ConfigMap
+    name: cni-${CLUSTER_NAME}-crs-0
+  strategy: ApplyOnce
+---
+apiVersion: v1
+data:
+  kindnet.yaml: |
+    # kindnetd networking manifest
+    ---
+    kind: ClusterRole
+    apiVersion: rbac.authorization.k8s.io/v1
+    metadata:
+      name: kindnet
+    rules:
+      - apiGroups:
+          - ""
+        resources:
+          - nodes
+        verbs:
+          - list
+          - watch
+          - patch
+      - apiGroups:
+          - ""
+        resources:
+          - configmaps
+        verbs:
+          - get
+    ---
+    kind: ClusterRoleBinding
+    apiVersion: rbac.authorization.k8s.io/v1
+    metadata:
+      name: kindnet
+    roleRef:
+      apiGroup: rbac.authorization.k8s.io
+      kind: ClusterRole
+      name: kindnet
+    subjects:
+      - kind: ServiceAccount
+        name: kindnet
+        namespace: kube-system
+    ---
+    apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+      name: kindnet
+      namespace: kube-system
+    ---
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: kindnet
+      namespace: kube-system
+      labels:
+        tier: node
+        app: kindnet
+        k8s-app: kindnet
+    spec:
+      selector:
+        matchLabels:
+          app: kindnet
+      template:
+        metadata:
+          labels:
+            tier: node
+            app: kindnet
+            k8s-app: kindnet
+        spec:
+          hostNetwork: true
+          tolerations:
+            - operator: Exists
+              effect: NoSchedule
+          serviceAccountName: kindnet
+          containers:
+            - name: kindnet-cni
+              image: kindest/kindnetd:v20230330-48f316cd
+              env:
+                - name: HOST_IP
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: status.hostIP
+                - name: POD_IP
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: status.podIP
+                - name: POD_SUBNET
+                  value: '192.168.0.0/16'
+              volumeMounts:
+                - name: cni-cfg
+                  mountPath: /etc/cni/net.d
+                - name: xtables-lock
+                  mountPath: /run/xtables.lock
+                  readOnly: false
+                - name: lib-modules
+                  mountPath: /lib/modules
+                  readOnly: true
+              resources:
+                requests:
+                  cpu: "100m"
+                  memory: "50Mi"
+                limits:
+                  cpu: "100m"
+                  memory: "50Mi"
+              securityContext:
+                privileged: false
+                capabilities:
+                  add: ["NET_RAW", "NET_ADMIN"]
+          volumes:
+            - name: cni-bin
+              hostPath:
+                path: /opt/cni/bin
+                type: DirectoryOrCreate
+            - name: cni-cfg
+              hostPath:
+                path: /etc/cni/net.d
+                type: DirectoryOrCreate
+            - name: xtables-lock
+              hostPath:
+                path: /run/xtables.lock
+                type: FileOrCreate
+            - name: lib-modules
+              hostPath:
+                path: /lib/modules
+kind: ConfigMap
+metadata:
+  name: cni-${CLUSTER_NAME}-crs-0