CNI plugin maintains the addnhosts file, dns resolves across hosts

hack/test-deployment.yaml

@@ -7,7 +7,7 @@ metadata:
   labels:
     app: nginx
 spec:
-  replicas: 0
+  replicas: 1
   selector:
     matchLabels:
       app: nginx
@@ -18,8 +18,6 @@ spec:
         app: nginx
     spec:
       containers:
-      - name: nginx
+      - name: nginx1
         image: nginx:1.14.2
-#        ports:
-#        - containerPort: 8000
 ---

manifests/coredns.yaml

@@ -17,26 +17,28 @@ spec:
     spec:
       hostNetwork: true
       volumes:
-      - name: skate
+      - name: cni
         hostPath:
-          path: /etc/skate
+          path: /var/lib/skatelet/cni/podman
       containers:
       - name: coredns
         image: ghcr.io/skateco/coredns
         volumeMounts:
-        - mountPath: /etc/skate
-          name: skate
+        - mountPath: /var/lib/skatelet/cni/podman
+          name: cni
         env:
         - name: CORE_FILE
           value: |
             cluster.skate:5553 {
+                debug
             
                 bind lo 0.0.0.0
             
-                hosts /run/containers/cni/dnsname/podman/addnhosts
+                hosts /var/lib/skatelet/cni/podman/addnhosts
             }
             
             cluster.skate:53 {
+                debug
             
                 bind lo 0.0.0.0
             
@@ -45,7 +47,7 @@ spec:
                 loadbalance round_robin
             
             }
-            . {
+            .:53 {
                 bind lo 0.0.0.0
                 forward . 8.8.8.8
                 cache

src/create.rs

@@ -11,10 +11,11 @@ use semver::{Version, VersionReq};
 use serde_json::to_string;
 use crate::apply::{apply, ApplyArgs};
 use crate::config::{Cluster, Config, Node};
+use crate::refresh::refreshed_state;
 use crate::skate::{ConfigFileArgs, Distribution, Os};
 
-use crate::ssh::{cluster_connections, node_connection, NodeSystemInfo, SshClient};
-use crate::state::state::ClusterState;
+use crate::ssh::{cluster_connections, node_connection, NodeSystemInfo, SshClient, SshClients};
+use crate::state::state::{ClusterState, NodeState};
 use crate::util::{CHECKBOX_EMOJI, CROSS_EMOJI};
 
 const COREDNS_MANIFEST: &str = include_str!("../manifests/coredns.yaml");
@@ -65,7 +66,7 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
     let context = match args.config.context {
         None => match config.current_context {
             None => {
-                Err(anyhow!("--cluster is required unless there is already a current context"))
+                Err(anyhow!("--context is required unless there is already a current context"))
             }
             Some(ref context) => Ok(context)
         }
@@ -75,8 +76,6 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
     let (cluster_index, cluster) = config.clusters.iter().find_position(|c| c.name == context.clone()).ok_or(anyhow!("no cluster by name of {}", context))?;
     let mut cluster = (*cluster).clone();
 
-    let state = ClusterState::load(cluster.name.as_str())?;
-
     let mut nodes_iter = cluster.nodes.clone().into_iter();
 
     let existing_index = nodes_iter.find_position(|n| n.name == args.name || n.host == args.host).map(|(p, _n)| p);
@@ -91,6 +90,7 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
         user: args.user.clone(),
         key: args.key.clone(),
         subnet_cidr: args.subnet_cidr.clone(),
+
     };
 
     match existing_index {
@@ -172,22 +172,59 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
     let cmd = "sudo podman image exists k8s.gcr.io/pause:3.5 || sudo podman pull  k8s.gcr.io/pause:3.5";
     conn.execute(cmd).await?;
 
-    setup_networking(&conn, &cluster, &node, &info, &args).await?;
+    let (all_conns, _) = cluster_connections(&cluster).await;
+    let all_conns = &all_conns.unwrap_or(SshClients { clients: vec!() });
 
 
-    config.persist(Some(args.config.skateconfig))?;
+    setup_networking(&conn, &all_conns, &cluster, &node).await?;
+
+    config.persist(Some(args.config.skateconfig.clone()))?;
 
+    /// Refresh state so that we can apply coredns later
+    let state = refreshed_state(&cluster.name, &all_conns, &config).await?;
     state.persist()?;
 
+    install_manifests(&args, &cluster, &node).await?;
+
+    Ok(())
+}
+
+async fn install_manifests(args: &CreateNodeArgs, config: &Cluster, node: &Node) -> Result<(), Box<dyn Error>> {
+    /// COREDNS
+    /// coredns listens on port 53 and 5533
+    /// port 53 serves .cluster.skate by forwarding to all coredns instances on port 5553
+    /// uses fanout plugin
+
+    let coredns_yaml_path = "/tmp/skate-coredns.yaml";
+    let mut file = File::create(coredns_yaml_path)?;
+    // replace forward list in coredns config with that of other hosts
+    let fanout_list = config.nodes.iter().filter(|n| n.name != node.name).map(|n| n.host.clone() + ":5553").join(" ");
+
+    let coredns_yaml = COREDNS_MANIFEST.replace("%%fanout_list%%", &fanout_list);
+
+    file.write_all(coredns_yaml.as_bytes())?;
+
+
+    apply(ApplyArgs {
+        filename: vec![coredns_yaml_path.to_string()],
+        grace_period: 0,
+        config: args.config.clone(),
+    }).await?;
+
     Ok(())
 }
 
-async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, _info: &NodeSystemInfo, args: &CreateNodeArgs) -> Result<(), Box<dyn Error>> {
+async fn setup_networking(conn: &SshClient, all_conns: &SshClients, cluster_conf: &Cluster, node: &Node) -> Result<(), Box<dyn Error>> {
+    let cmd = "sqlite3 -version || sudo apt-get install -y sqlite3";
+    conn.execute(cmd).await?;
+
     let cmd = "sudo cp /usr/share/containers/containers.conf /etc/containers/containers.conf";
     conn.execute(cmd).await?;
 
     let cmd = format!("sudo sed -i 's&#default_subnet[ =].*&default_subnet = \"{}\"&' /etc/containers/containers.conf", node.subnet_cidr);
     conn.execute(&cmd).await?;
+    let cmd = "sudo sed -i 's&#network_backend[ =].*&network_backend = \"cni\"&' /etc/containers/containers.conf";
+    conn.execute(&cmd).await?;
 
     let cmd = "sudo ip link del cni-podman0|| exit 0";
     conn.execute(&cmd).await?;
@@ -227,14 +264,8 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node,
     conn.execute(cmd).await?;
 
 
-    let (conns, _errs) = cluster_connections(cluster_conf).await;
-    match conns {
-        Some(conns) => {
-            for conn in conns.clients {
-                create_replace_routes_file(&conn, cluster_conf).await?;
-            }
-        }
-        _ => {}
+    for conn in &all_conns.clients {
+        create_replace_routes_file(conn, cluster_conf).await?;
     }
 
     let cmd = "sudo podman image exists ghcr.io/skateco/coredns || sudo podman pull ghcr.io/skateco/coredns";
@@ -243,10 +274,18 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node,
 
     // In ubuntu 24.04 there's an issue with apparmor and podman
     // https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2040483
-    let cmd = "sudo systemctl disable apparmor.service --now";
-    conn.execute(cmd).await?;
+
+    let cmd = "sudo systemctl list-unit-files apparmor.service";
+    let apparmor_unit_exists = conn.execute(cmd).await;
+
+    if apparmor_unit_exists.is_ok() {
+        let cmd = "sudo systemctl disable apparmor.service --now";
+        conn.execute(cmd).await?;
+    }
     let cmd = "sudo aa-teardown";
     _ = conn.execute(cmd).await;
+    let cmd = "sudo apt purge -y apparmor";
+    _ = conn.execute(cmd).await;
 
 
     // // install dnsmasq
@@ -257,27 +296,7 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node,
     conn.execute(cmd).await?;
     // changed /etc/resolv.conf to be 127.0.0.1
     let cmd = "sudo bash -c 'echo 127.0.0.1 > /etc/resolv.conf'";
-    conn.execute(cmd).await?;
-
-    /// COREDNS
-    /// coredns listens on port 53 and 5533
-    /// port 53 serves .cluster.skate by forwarding to all coredns instances on port 5553
-    /// uses fanout plugin
-    let coredns_yaml_path = "/tmp/skate-coredns.yaml";
-    let mut file = File::create(coredns_yaml_path)?;
-    // replace forward list in coredns config with that of other hosts
-    let fanout_list = cluster_conf.nodes.iter().filter(|n| n.name != node.name).map(|n| n.host.clone() + ":5553").join(" ");
-
-    let coredns_yaml = COREDNS_MANIFEST.replace("%%fanout_list%%", &fanout_list);
-
-    file.write_all(coredns_yaml.as_bytes())?;
-
-
-    apply(ApplyArgs {
-        filename: vec![coredns_yaml_path.to_string()],
-        grace_period: 0,
-        config: args.config.clone(),
-    }).await?;
+    _ = conn.execute(cmd).await;
 
 
 //     let dnsmasq_conf = general_purpose::STANDARD.encode("

src/delete.rs

@@ -39,7 +39,7 @@ async fn delete_node(args: DeleteNodeArgs) -> Result<(), Box<dyn Error>> {
     let context = match args.config.context {
         None => match config.current_context {
             None => {
-                Err(anyhow!("--cluster is required unless there is already a current context"))
+                Err(anyhow!("--context is required unless there is already a current context"))
             }
             Some(ref context) => Ok(context)
         }
@@ -51,7 +51,7 @@ async fn delete_node(args: DeleteNodeArgs) -> Result<(), Box<dyn Error>> {
     let find_result = cluster.nodes.iter().find_position(|n| n.name == args.name);
 
     match find_result {
-        Some((p,_)) => {
+        Some((p, _)) => {
             config.clusters[cluster_index].nodes.remove(p);
             config.persist(Some(args.config.skateconfig))
         }

src/executor.rs

@@ -28,16 +28,21 @@ impl Executor for DefaultExecutor {
         // just to check
         let object: SupportedResources = serde_yaml::from_str(manifest).expect("failed to deserialize manifest");
 
+        // check if object's hostNetwork: true then don't use network=podman
 
         let file_path = DefaultExecutor::write_to_file(&serde_yaml::to_string(&object)?)?;
 
-        let args = ["play", "kube", &file_path, "--start"];
+        let mut args = vec!["play", "kube", &file_path, "--start"];
+        if !object.host_network() {
+            args.push("--network=podman")
+        }
+
         let output = process::Command::new("podman")
-            .args(args)
+            .args(&args)
             .stdin(Stdio::piped())
             .stdout(Stdio::piped())
             .output()
-            .expect(&format!("failed to apply resource via `podman {}`", args.join(" ")));
+            .expect(&format!("failed to apply resource via `podman {}`", &args.join(" ")));
 
         if !output.status.success() {
             return Err(anyhow!("`podman {}` exited with code {}, stderr: {}", args.join(" "), output.status.code().unwrap(), String::from_utf8_lossy(&output.stderr).to_string()).into());

src/scheduler.rs

@@ -1,4 +1,3 @@
-
 use std::cmp::Ordering;
 use std::collections::BTreeMap;
 use std::error::Error;
@@ -52,8 +51,13 @@ pub struct ApplyPlan {
     pub actions: Vec<ScheduledOperation<SupportedResources>>,
 }
 
+pub struct RejectedNode {
+    pub node_name: String,
+    pub reason: String,
+}
+
 impl DefaultScheduler {
-    fn choose_node(nodes: Vec<NodeState>, object: &SupportedResources) -> Option<NodeState> {
+    fn choose_node(nodes: Vec<NodeState>, object: &SupportedResources) -> (Option<NodeState>, Vec<RejectedNode>) {
         // filter nodes based on resource requirements  - cpu, memory, etc
 
         let node_selector = match object {
@@ -65,20 +69,36 @@ impl DefaultScheduler {
             _ => None
         }.unwrap_or(BTreeMap::new());
 
+        let mut rejected_nodes: Vec<RejectedNode> = vec!();
 
 
         let filtered_nodes = nodes.iter().filter(|n| {
             let k8s_node: K8sNode = (**n).clone().into();
             let node_labels = k8s_node.metadata.labels.unwrap_or_default();
             // only schedulable nodes
-            k8s_node.spec.and_then(|s| {
+            let is_schedulable = k8s_node.spec.and_then(|s| {
                 s.unschedulable.and_then(|u| Some(!u))
-            }).unwrap_or(false)
-                &&
-                // only nodes that match the nodeselectors
-                node_selector.iter().all(|(k, v)| {
-                    node_labels.get(k).unwrap_or(&"".to_string()) == v
-                })
+            }).unwrap_or(false);
+
+            if !is_schedulable {
+                rejected_nodes.push(RejectedNode {
+                    node_name: n.node_name.clone(),
+                    reason: "node is unschedulable".to_string(),
+                });
+                return false;
+            }
+
+            // only nodes that match the nodeselectors
+            node_selector.iter().all(|(k, v)| {
+                let matches = node_labels.get(k).unwrap_or(&"".to_string()) == v;
+                if (!matches) {
+                    rejected_nodes.push(RejectedNode {
+                        node_name: n.node_name.clone(),
+                        reason: format!("node selector {}:{} did not match", k, v),
+                    });
+                }
+                return matches;
+            })
         }).collect::<Vec<_>>();
 
 
@@ -104,7 +124,7 @@ impl DefaultScheduler {
             }).or_else(|| Some(node.clone()))
         });
 
-        feasible_node
+        (feasible_node, rejected_nodes)
     }
 
     fn plan_daemonset(state: &ClusterState, ds: &DaemonSet) -> Result<ApplyPlan, Box<dyn Error>> {
@@ -128,7 +148,7 @@ impl DefaultScheduler {
             // bind to specific node
             pod_spec.node_selector = Some({
                 let mut selector = pod_spec.node_selector.unwrap_or_default();
-                selector.insert("skate.io/hostname".to_string(), node_name.clone());
+                selector.insert("skate.io/nodename".to_string(), node_name.clone());
                 selector
             });
 
@@ -351,7 +371,14 @@ impl DefaultScheduler {
                     }
                 }
                 OpType::Create => {
-                    let node_name = Self::choose_node(state.nodes.clone(), &action.resource).ok_or("failed to find feasible node")?.node_name.clone();
+                    let (node, rejected_nodes) = Self::choose_node(state.nodes.clone(), &action.resource);
+                    if !node.is_some() {
+                        let reasons = rejected_nodes.iter().map(|r| format!("{} - {}", r.node_name, r.reason)).collect::<Vec<_>>().join(", ");
+                        return Err(anyhow!("failed to find feasible node: {}", reasons).into());
+                    }
+
+                    let node_name = node.unwrap().node_name.clone();
+
                     let client = conns.find(&node_name).unwrap();
                     let serialized = serde_yaml::to_string(&action.resource).expect("failed to serialize object");
 

src/skate.rs

@@ -100,6 +100,15 @@ impl SupportedResources {
             SupportedResources::DaemonSet(d) => metadata_name(d),
         }
     }
+
+    // whether there's host network set
+    pub fn host_network(&self) -> bool {
+        match self {
+            SupportedResources::Pod(p) => p.clone().spec.unwrap().host_network.unwrap(),
+            SupportedResources::Deployment(d) => d.clone().spec.unwrap().template.spec.unwrap().host_network.unwrap(),
+            SupportedResources::DaemonSet(d) => d.clone().spec.unwrap().template.spec.unwrap().host_network.unwrap(),
+        }
+    }
     fn fixup_metadata(meta: ObjectMeta, extra_labels: Option<HashMap<String, String>>) -> Result<ObjectMeta, Box<dyn Error>> {
         let mut meta = meta.clone();
         let ns = meta.namespace.clone().unwrap_or("default".to_string());