Merge pull request #461 from flatcar/tormath1/cilium-selinux

kubeadm: add logic to enforce SELinux for Cilium CNI in Flatcar >= 3745
flatcar · Oct 10, 2023 · 96843d2 · 96843d2
2 parents fbb5dee + daead39
commit 96843d2
Show file tree

Hide file tree

Showing 15 changed files with 50 additions and 15 deletions.
diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go
@@ -54,8 +54,12 @@ var (
 					_ = c.MustSSH(controller, "/opt/bin/cilium uninstall")
 					version := params["CiliumVersion"].(string)
 					cidr := params["PodSubnet"].(string)
-					cmd := fmt.Sprintf("/opt/bin/cilium install --config enable-endpoint-routes=true --config cluster-pool-ipv4-cidr=%s --version=%s --encryption=ipsec --wait --wait-duration 1m", cidr, version)
-					_ = c.MustSSH(controller, cmd)
+					cmd := fmt.Sprintf("/opt/bin/cilium install --config enable-endpoint-routes=true --config cluster-pool-ipv4-cidr=%s --version=%s --encryption=ipsec --wait=false --restart-unmanaged-pods=false --rollback=false", cidr, version)
+					_, _ = c.SSH(controller, cmd)
+					patch := `{ grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && /opt/bin/kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true`
+					_ = c.MustSSH(controller, patch)
+					status := "/opt/bin/cilium status --wait --wait-duration 1m"
+					_ = c.MustSSH(controller, status)
 				},
 			},
 		},
@@ -368,6 +372,32 @@ func setup(c cluster.TestCluster, params map[string]interface{}) (platform.Machi
 		return nil, fmt.Errorf("unable to create etcd node: %w", err)
 	}
 
+	v := string(c.MustSSH(etcdNode, `set -euo pipefail; grep -m 1 "^VERSION=" /usr/lib/os-release | cut -d = -f 2`))
+	if v == "" {
+		c.Fatalf("Assertion for version string failed")
+	}
+
+	version, err := semver.NewVersion(v)
+	if err != nil {
+		c.Fatalf("unable to create semver version from %s: %v", version, err)
+	}
+
+	// For Cilium CNI, we enforce SELinux only for version >= 3745 because the SELinux policies update (container_t/spc_t) is not yet
+	// propagated through all the channels.
+	// The etcd node will run with enforced SELinux anyway but we want to test SELinux on the worker / master nodes.
+	cni, ok := params["CNI"]
+	if !ok {
+		c.Fatal("unable to get CNI value")
+	}
+
+	if cni == "cilium" && version.LessThan(semver.Version{Major: 3745}) {
+		r := c.RuntimeConf()
+		if r != nil {
+			plog.Infof("Setting SELinux to permissive mode")
+			r.NoEnableSelinux = true
+		}
+	}
+
 	if err := etcd.GetClusterHealth(c, etcdNode, 1); err != nil {
 		return nil, fmt.Errorf("unable to get etcd node health: %w", err)
 	}

diff --git a/kola/tests/kubeadm/templates.go b/kola/tests/kubeadm/templates.go
@@ -403,6 +403,7 @@ EOF
         --config enable-endpoint-routes=true \
         --config cluster-pool-ipv4-cidr={{ .PodSubnet }} \
         --version={{ .CiliumVersion }} 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT
+    { grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true
     # --wait will wait for status to report success
     /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT
 {{ end }}

diff --git a/kola/tests/kubeadm/testdata/master-cilium-script.sh b/kola/tests/kubeadm/testdata/master-cilium-script.sh
@@ -91,6 +91,7 @@ EOF
         --config enable-endpoint-routes=true \
         --config cluster-pool-ipv4-cidr=192.168.0.0/17 \
         --version=v0.11.1 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT
+    { grep -q svirt_lxc_file_t /etc/selinux/mcs/contexts/lxc_contexts && kubectl --namespace kube-system patch daemonset/cilium -p '{"spec":{"template":{"spec":{"containers":[{"name":"cilium-agent","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}],"initContainers":[{"name":"mount-cgroup","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"apply-sysctl-overwrites","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}},{"name":"clean-cilium-state","securityContext":{"seLinuxOptions":{"level":"s0","type":"unconfined_t"}}}]}}}}'; } || true
     # --wait will wait for status to report success
     /opt/bin/cilium status --wait 2>&1 | iconv --from-code utf-8 --to-code ascii//TRANSLIT
 

diff --git a/platform/cluster.go b/platform/cluster.go
@@ -283,8 +283,8 @@ func (bc *BaseCluster) Name() string {
 	return bc.name
 }
 
-func (bc *BaseCluster) RuntimeConf() RuntimeConfig {
-	return *bc.rconf
+func (bc *BaseCluster) RuntimeConf() *RuntimeConfig {
+	return bc.rconf
 }
 
 func (bc *BaseCluster) ConsoleOutput() map[string]string {

diff --git a/platform/machine/aws/machine.go b/platform/machine/aws/machine.go
@@ -48,7 +48,7 @@ func (am *machine) PrivateIP() string {
 	return *am.mach.PrivateIpAddress
 }
 
-func (am *machine) RuntimeConf() platform.RuntimeConfig {
+func (am *machine) RuntimeConf() *platform.RuntimeConfig {
 	return am.cluster.RuntimeConf()
 }
 

diff --git a/platform/machine/azure/machine.go b/platform/machine/azure/machine.go
@@ -45,7 +45,7 @@ func (am *machine) PrivateIP() string {
 	return am.mach.PrivateIPAddress
 }
 
-func (am *machine) RuntimeConf() platform.RuntimeConfig {
+func (am *machine) RuntimeConf() *platform.RuntimeConfig {
 	return am.cluster.RuntimeConf()
 }
 

diff --git a/platform/machine/do/machine.go b/platform/machine/do/machine.go
@@ -44,7 +44,7 @@ func (dm *machine) PrivateIP() string {
 	return dm.privateIP
 }
 
-func (dm *machine) RuntimeConf() platform.RuntimeConfig {
+func (dm *machine) RuntimeConf() *platform.RuntimeConfig {
 	return dm.cluster.RuntimeConf()
 }
 

diff --git a/platform/machine/equinixmetal/machine.go b/platform/machine/equinixmetal/machine.go
@@ -44,7 +44,7 @@ func (pm *machine) PrivateIP() string {
 	return pm.privateIP
 }
 
-func (pm *machine) RuntimeConf() platform.RuntimeConfig {
+func (pm *machine) RuntimeConf() *platform.RuntimeConfig {
 	return pm.cluster.RuntimeConf()
 }
 

diff --git a/platform/machine/esx/machine.go b/platform/machine/esx/machine.go
@@ -46,7 +46,7 @@ func (em *machine) PrivateIP() string {
 	return em.mach.IPAddress
 }
 
-func (em *machine) RuntimeConf() platform.RuntimeConfig {
+func (em *machine) RuntimeConf() *platform.RuntimeConfig {
 	return em.cluster.RuntimeConf()
 }
 

diff --git a/platform/machine/external/machine.go b/platform/machine/external/machine.go
@@ -40,7 +40,7 @@ func (pm *machine) PrivateIP() string {
 	return pm.ipAddr
 }
 
-func (pm *machine) RuntimeConf() platform.RuntimeConfig {
+func (pm *machine) RuntimeConf() *platform.RuntimeConfig {
 	return pm.cluster.RuntimeConf()
 }
 

diff --git a/platform/machine/gcloud/machine.go b/platform/machine/gcloud/machine.go
@@ -45,7 +45,7 @@ func (gm *machine) PrivateIP() string {
 	return gm.intIP
 }
 
-func (gm *machine) RuntimeConf() platform.RuntimeConfig {
+func (gm *machine) RuntimeConf() *platform.RuntimeConfig {
 	return gm.gc.RuntimeConf()
 }
 

diff --git a/platform/machine/openstack/machine.go b/platform/machine/openstack/machine.go
@@ -103,7 +103,7 @@ func (om *machine) PrivateIP() string {
 	return om.IP()
 }
 
-func (om *machine) RuntimeConf() platform.RuntimeConfig {
+func (om *machine) RuntimeConf() *platform.RuntimeConfig {
 	return om.cluster.RuntimeConf()
 }
 

diff --git a/platform/machine/qemu/machine.go b/platform/machine/qemu/machine.go
@@ -46,7 +46,7 @@ func (m *machine) PrivateIP() string {
 	return m.netif.DHCPv4[0].IP.String()
 }
 
-func (m *machine) RuntimeConf() platform.RuntimeConfig {
+func (m *machine) RuntimeConf() *platform.RuntimeConfig {
 	return m.qc.RuntimeConf()
 }
 

diff --git a/platform/machine/unprivqemu/machine.go b/platform/machine/unprivqemu/machine.go
@@ -46,7 +46,7 @@ func (m *machine) PrivateIP() string {
 	return m.privateAddr
 }
 
-func (m *machine) RuntimeConf() platform.RuntimeConfig {
+func (m *machine) RuntimeConf() *platform.RuntimeConfig {
 	return m.qc.RuntimeConf()
 }
 

diff --git a/platform/platform.go b/platform/platform.go
@@ -50,7 +50,7 @@ type Machine interface {
 	PrivateIP() string
 
 	// RuntimeConf returns the cluster's runtime configuration.
-	RuntimeConf() RuntimeConfig
+	RuntimeConf() *RuntimeConfig
 
 	// SSHClient establishes a new SSH connection to the machine.
 	SSHClient() (*ssh.Client, error)
@@ -113,6 +113,9 @@ type Cluster interface {
 	// IgnitionVersion returns the version of Ignition supported by the
 	// cluster
 	IgnitionVersion() string
+
+	// RuntimeConf returns a pointer to the runtime configuration.
+	RuntimeConf() *RuntimeConfig
 }
 
 // Flight represents a group of Clusters within a single platform.