diff --git a/hack/test-deployment.yaml b/hack/test-deployment.yaml index 624b5b0..6df324c 100644 --- a/hack/test-deployment.yaml +++ b/hack/test-deployment.yaml @@ -7,7 +7,7 @@ metadata: labels: app: nginx spec: - replicas: 0 + replicas: 1 selector: matchLabels: app: nginx @@ -18,8 +18,6 @@ spec: app: nginx spec: containers: - - name: nginx + - name: nginx1 image: nginx:1.14.2 -# ports: -# - containerPort: 8000 --- diff --git a/manifests/coredns.yaml b/manifests/coredns.yaml index bff1f50..216657e 100644 --- a/manifests/coredns.yaml +++ b/manifests/coredns.yaml @@ -17,26 +17,28 @@ spec: spec: hostNetwork: true volumes: - - name: skate + - name: cni hostPath: - path: /etc/skate + path: /var/lib/skatelet/cni/podman containers: - name: coredns image: ghcr.io/skateco/coredns volumeMounts: - - mountPath: /etc/skate - name: skate + - mountPath: /var/lib/skatelet/cni/podman + name: cni env: - name: CORE_FILE value: | cluster.skate:5553 { + debug bind lo 0.0.0.0 - hosts /run/containers/cni/dnsname/podman/addnhosts + hosts /var/lib/skatelet/cni/podman/addnhosts } cluster.skate:53 { + debug bind lo 0.0.0.0 @@ -45,7 +47,7 @@ spec: loadbalance round_robin } - . { + .:53 { bind lo 0.0.0.0 forward . 8.8.8.8 cache diff --git a/src/create.rs b/src/create.rs index c725e00..d8598bc 100644 --- a/src/create.rs +++ b/src/create.rs @@ -11,10 +11,11 @@ use semver::{Version, VersionReq}; use serde_json::to_string; use crate::apply::{apply, ApplyArgs}; use crate::config::{Cluster, Config, Node}; +use crate::refresh::refreshed_state; use crate::skate::{ConfigFileArgs, Distribution, Os}; -use crate::ssh::{cluster_connections, node_connection, NodeSystemInfo, SshClient}; -use crate::state::state::ClusterState; +use crate::ssh::{cluster_connections, node_connection, NodeSystemInfo, SshClient, SshClients}; +use crate::state::state::{ClusterState, NodeState}; use crate::util::{CHECKBOX_EMOJI, CROSS_EMOJI}; const COREDNS_MANIFEST: &str = include_str!("../manifests/coredns.yaml"); @@ -65,7 +66,7 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box> { let context = match args.config.context { None => match config.current_context { None => { - Err(anyhow!("--cluster is required unless there is already a current context")) + Err(anyhow!("--context is required unless there is already a current context")) } Some(ref context) => Ok(context) } @@ -75,8 +76,6 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box> { let (cluster_index, cluster) = config.clusters.iter().find_position(|c| c.name == context.clone()).ok_or(anyhow!("no cluster by name of {}", context))?; let mut cluster = (*cluster).clone(); - let state = ClusterState::load(cluster.name.as_str())?; - let mut nodes_iter = cluster.nodes.clone().into_iter(); let existing_index = nodes_iter.find_position(|n| n.name == args.name || n.host == args.host).map(|(p, _n)| p); @@ -91,6 +90,7 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box> { user: args.user.clone(), key: args.key.clone(), subnet_cidr: args.subnet_cidr.clone(), + }; match existing_index { @@ -172,22 +172,59 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box> { let cmd = "sudo podman image exists k8s.gcr.io/pause:3.5 || sudo podman pull k8s.gcr.io/pause:3.5"; conn.execute(cmd).await?; - setup_networking(&conn, &cluster, &node, &info, &args).await?; + let (all_conns, _) = cluster_connections(&cluster).await; + let all_conns = &all_conns.unwrap_or(SshClients { clients: vec!() }); - config.persist(Some(args.config.skateconfig))?; + setup_networking(&conn, &all_conns, &cluster, &node).await?; + + config.persist(Some(args.config.skateconfig.clone()))?; + /// Refresh state so that we can apply coredns later + let state = refreshed_state(&cluster.name, &all_conns, &config).await?; state.persist()?; + install_manifests(&args, &cluster, &node).await?; + + Ok(()) +} + +async fn install_manifests(args: &CreateNodeArgs, config: &Cluster, node: &Node) -> Result<(), Box> { + /// COREDNS + /// coredns listens on port 53 and 5533 + /// port 53 serves .cluster.skate by forwarding to all coredns instances on port 5553 + /// uses fanout plugin + + let coredns_yaml_path = "/tmp/skate-coredns.yaml"; + let mut file = File::create(coredns_yaml_path)?; + // replace forward list in coredns config with that of other hosts + let fanout_list = config.nodes.iter().filter(|n| n.name != node.name).map(|n| n.host.clone() + ":5553").join(" "); + + let coredns_yaml = COREDNS_MANIFEST.replace("%%fanout_list%%", &fanout_list); + + file.write_all(coredns_yaml.as_bytes())?; + + + apply(ApplyArgs { + filename: vec![coredns_yaml_path.to_string()], + grace_period: 0, + config: args.config.clone(), + }).await?; + Ok(()) } -async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, _info: &NodeSystemInfo, args: &CreateNodeArgs) -> Result<(), Box> { +async fn setup_networking(conn: &SshClient, all_conns: &SshClients, cluster_conf: &Cluster, node: &Node) -> Result<(), Box> { + let cmd = "sqlite3 -version || sudo apt-get install -y sqlite3"; + conn.execute(cmd).await?; + let cmd = "sudo cp /usr/share/containers/containers.conf /etc/containers/containers.conf"; conn.execute(cmd).await?; let cmd = format!("sudo sed -i 's&#default_subnet[ =].*&default_subnet = \"{}\"&' /etc/containers/containers.conf", node.subnet_cidr); conn.execute(&cmd).await?; + let cmd = "sudo sed -i 's&#network_backend[ =].*&network_backend = \"cni\"&' /etc/containers/containers.conf"; + conn.execute(&cmd).await?; let cmd = "sudo ip link del cni-podman0|| exit 0"; conn.execute(&cmd).await?; @@ -227,14 +264,8 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, conn.execute(cmd).await?; - let (conns, _errs) = cluster_connections(cluster_conf).await; - match conns { - Some(conns) => { - for conn in conns.clients { - create_replace_routes_file(&conn, cluster_conf).await?; - } - } - _ => {} + for conn in &all_conns.clients { + create_replace_routes_file(conn, cluster_conf).await?; } let cmd = "sudo podman image exists ghcr.io/skateco/coredns || sudo podman pull ghcr.io/skateco/coredns"; @@ -243,10 +274,18 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, // In ubuntu 24.04 there's an issue with apparmor and podman // https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2040483 - let cmd = "sudo systemctl disable apparmor.service --now"; - conn.execute(cmd).await?; + + let cmd = "sudo systemctl list-unit-files apparmor.service"; + let apparmor_unit_exists = conn.execute(cmd).await; + + if apparmor_unit_exists.is_ok() { + let cmd = "sudo systemctl disable apparmor.service --now"; + conn.execute(cmd).await?; + } let cmd = "sudo aa-teardown"; _ = conn.execute(cmd).await; + let cmd = "sudo apt purge -y apparmor"; + _ = conn.execute(cmd).await; // // install dnsmasq @@ -257,27 +296,7 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, conn.execute(cmd).await?; // changed /etc/resolv.conf to be 127.0.0.1 let cmd = "sudo bash -c 'echo 127.0.0.1 > /etc/resolv.conf'"; - conn.execute(cmd).await?; - - /// COREDNS - /// coredns listens on port 53 and 5533 - /// port 53 serves .cluster.skate by forwarding to all coredns instances on port 5553 - /// uses fanout plugin - let coredns_yaml_path = "/tmp/skate-coredns.yaml"; - let mut file = File::create(coredns_yaml_path)?; - // replace forward list in coredns config with that of other hosts - let fanout_list = cluster_conf.nodes.iter().filter(|n| n.name != node.name).map(|n| n.host.clone() + ":5553").join(" "); - - let coredns_yaml = COREDNS_MANIFEST.replace("%%fanout_list%%", &fanout_list); - - file.write_all(coredns_yaml.as_bytes())?; - - - apply(ApplyArgs { - filename: vec![coredns_yaml_path.to_string()], - grace_period: 0, - config: args.config.clone(), - }).await?; + _ = conn.execute(cmd).await; // let dnsmasq_conf = general_purpose::STANDARD.encode(" diff --git a/src/delete.rs b/src/delete.rs index bc839a8..9dc99e6 100644 --- a/src/delete.rs +++ b/src/delete.rs @@ -39,7 +39,7 @@ async fn delete_node(args: DeleteNodeArgs) -> Result<(), Box> { let context = match args.config.context { None => match config.current_context { None => { - Err(anyhow!("--cluster is required unless there is already a current context")) + Err(anyhow!("--context is required unless there is already a current context")) } Some(ref context) => Ok(context) } @@ -51,7 +51,7 @@ async fn delete_node(args: DeleteNodeArgs) -> Result<(), Box> { let find_result = cluster.nodes.iter().find_position(|n| n.name == args.name); match find_result { - Some((p,_)) => { + Some((p, _)) => { config.clusters[cluster_index].nodes.remove(p); config.persist(Some(args.config.skateconfig)) } diff --git a/src/executor.rs b/src/executor.rs index 4b80315..cac8b26 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -28,16 +28,21 @@ impl Executor for DefaultExecutor { // just to check let object: SupportedResources = serde_yaml::from_str(manifest).expect("failed to deserialize manifest"); + // check if object's hostNetwork: true then don't use network=podman let file_path = DefaultExecutor::write_to_file(&serde_yaml::to_string(&object)?)?; - let args = ["play", "kube", &file_path, "--start"]; + let mut args = vec!["play", "kube", &file_path, "--start"]; + if !object.host_network() { + args.push("--network=podman") + } + let output = process::Command::new("podman") - .args(args) + .args(&args) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .output() - .expect(&format!("failed to apply resource via `podman {}`", args.join(" "))); + .expect(&format!("failed to apply resource via `podman {}`", &args.join(" "))); if !output.status.success() { return Err(anyhow!("`podman {}` exited with code {}, stderr: {}", args.join(" "), output.status.code().unwrap(), String::from_utf8_lossy(&output.stderr).to_string()).into()); diff --git a/src/scheduler.rs b/src/scheduler.rs index 9e845cc..2780884 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -1,4 +1,3 @@ - use std::cmp::Ordering; use std::collections::BTreeMap; use std::error::Error; @@ -52,8 +51,13 @@ pub struct ApplyPlan { pub actions: Vec>, } +pub struct RejectedNode { + pub node_name: String, + pub reason: String, +} + impl DefaultScheduler { - fn choose_node(nodes: Vec, object: &SupportedResources) -> Option { + fn choose_node(nodes: Vec, object: &SupportedResources) -> (Option, Vec) { // filter nodes based on resource requirements - cpu, memory, etc let node_selector = match object { @@ -65,20 +69,36 @@ impl DefaultScheduler { _ => None }.unwrap_or(BTreeMap::new()); + let mut rejected_nodes: Vec = vec!(); let filtered_nodes = nodes.iter().filter(|n| { let k8s_node: K8sNode = (**n).clone().into(); let node_labels = k8s_node.metadata.labels.unwrap_or_default(); // only schedulable nodes - k8s_node.spec.and_then(|s| { + let is_schedulable = k8s_node.spec.and_then(|s| { s.unschedulable.and_then(|u| Some(!u)) - }).unwrap_or(false) - && - // only nodes that match the nodeselectors - node_selector.iter().all(|(k, v)| { - node_labels.get(k).unwrap_or(&"".to_string()) == v - }) + }).unwrap_or(false); + + if !is_schedulable { + rejected_nodes.push(RejectedNode { + node_name: n.node_name.clone(), + reason: "node is unschedulable".to_string(), + }); + return false; + } + + // only nodes that match the nodeselectors + node_selector.iter().all(|(k, v)| { + let matches = node_labels.get(k).unwrap_or(&"".to_string()) == v; + if (!matches) { + rejected_nodes.push(RejectedNode { + node_name: n.node_name.clone(), + reason: format!("node selector {}:{} did not match", k, v), + }); + } + return matches; + }) }).collect::>(); @@ -104,7 +124,7 @@ impl DefaultScheduler { }).or_else(|| Some(node.clone())) }); - feasible_node + (feasible_node, rejected_nodes) } fn plan_daemonset(state: &ClusterState, ds: &DaemonSet) -> Result> { @@ -128,7 +148,7 @@ impl DefaultScheduler { // bind to specific node pod_spec.node_selector = Some({ let mut selector = pod_spec.node_selector.unwrap_or_default(); - selector.insert("skate.io/hostname".to_string(), node_name.clone()); + selector.insert("skate.io/nodename".to_string(), node_name.clone()); selector }); @@ -351,7 +371,14 @@ impl DefaultScheduler { } } OpType::Create => { - let node_name = Self::choose_node(state.nodes.clone(), &action.resource).ok_or("failed to find feasible node")?.node_name.clone(); + let (node, rejected_nodes) = Self::choose_node(state.nodes.clone(), &action.resource); + if !node.is_some() { + let reasons = rejected_nodes.iter().map(|r| format!("{} - {}", r.node_name, r.reason)).collect::>().join(", "); + return Err(anyhow!("failed to find feasible node: {}", reasons).into()); + } + + let node_name = node.unwrap().node_name.clone(); + let client = conns.find(&node_name).unwrap(); let serialized = serde_yaml::to_string(&action.resource).expect("failed to serialize object"); diff --git a/src/skate.rs b/src/skate.rs index 2e77b68..1f0f4fb 100644 --- a/src/skate.rs +++ b/src/skate.rs @@ -100,6 +100,15 @@ impl SupportedResources { SupportedResources::DaemonSet(d) => metadata_name(d), } } + + // whether there's host network set + pub fn host_network(&self) -> bool { + match self { + SupportedResources::Pod(p) => p.clone().spec.unwrap().host_network.unwrap(), + SupportedResources::Deployment(d) => d.clone().spec.unwrap().template.spec.unwrap().host_network.unwrap(), + SupportedResources::DaemonSet(d) => d.clone().spec.unwrap().template.spec.unwrap().host_network.unwrap(), + } + } fn fixup_metadata(meta: ObjectMeta, extra_labels: Option>) -> Result> { let mut meta = meta.clone(); let ns = meta.namespace.clone().unwrap_or("default".to_string()); diff --git a/src/skatelet/cni.rs b/src/skatelet/cni.rs index 59cdf02..a81a84e 100644 --- a/src/skatelet/cni.rs +++ b/src/skatelet/cni.rs @@ -14,15 +14,15 @@ use anyhow::anyhow; use cni_plugin::config::NetworkConfig; use serde_json::Value; use serde_json::Value::String as JsonString; +use crate::skate::exec_cmd; - -fn conf_path() -> String { +fn conf_path_str() -> String { "/var/lib/skatelet/cni".to_string() } fn lock(network_name: &str, cb: &dyn Fn() -> Result>) -> Result> { - let lock_path = Path::new(&conf_path()).join(network_name).join("lock"); + let lock_path = Path::new(&conf_path_str()).join(network_name).join("lock"); let lock_file = File::create(lock_path.clone()).map_err(|e| anyhow!("failed to create/open lock file: {}", e))?; debug!("waiting for lock on {}", lock_path.display()); lock_file.lock_exclusive()?; @@ -35,10 +35,10 @@ fn lock(network_name: &str, cb: &dyn Fn() -> Result>) -> Re result } -fn ensure_paths(net_name: &str) { - let conf_path_str = conf_path(); - let conf_path = Path::new(&conf_path_str); - let net_path = conf_path.join(net_name); +fn ensure_skatelet_cni_conf_dir(dir_name: &str) { + let conf_str = conf_path_str(); + let conf_path = Path::new(&conf_str); + let net_path = conf_path.join(dir_name); fs::create_dir_all(conf_path).unwrap(); fs::create_dir_all(net_path).unwrap(); @@ -89,45 +89,121 @@ fn extract_prev_result(prev_value: Option) -> Option { }) } +fn write_last_run_file(msg: &str) { + let last_err_path = Path::new(&conf_path_str()).join(".last_run.log"); + let mut last_err_file = OpenOptions::new() + .create(true) + .write(true) + .append(true) + .open(last_err_path).unwrap(); + writeln!(last_err_file, "{}", msg).unwrap(); +} + pub fn cni() { match run() { - Ok(_) => {} - Err(_e) => { - // handle error formatting + Ok(warning) => { + write_last_run_file(&format!("WARNING: {}", warning)) + } + Err(e) => { + write_last_run_file(&format!("ERROR: {}", e)); + panic!("{}", e) } } } -fn run() -> Result<(), Box> { +fn run() -> Result> { + let conf_str = conf_path_str(); + let conf_path = Path::new(&conf_str); + + fs::create_dir_all(conf_path).unwrap(); + let cmd = var("CNI_COMMAND").unwrap_or_default(); match cmd.as_str() { "ADD" => { let json: Value = serde_json::from_reader(io::stdin()).map_err(|e| anyhow!("failed to parse stdin: {}", e))?; - if !json["prevResult"].is_object() { - return Err(anyhow!("failed to parse prevResult").into()); + + let config: NetworkConfig = serde_json::from_value(json.clone()).map_err(|e| anyhow!("failed to parse config: {}", e))?; + + let result = prev_result_or_default(&config); + + if result.ips.len() == 0 { + return Err("no ips in prev_result".into()); } - let prev_result = json["prevResult"].clone(); - let output = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; - print!("{}", output); + + + let container_id = var("CNI_CONTAINERID")?; + + // get podman info from sqlitedb in /var/lib/containers/storage/db.sql + let pod_json = exec_cmd( + "sqlite3", + &[ + "/var/lib/containers/storage/db.sql", + &format!("select p.json from ContainerConfig c join PodConfig p on c.PodID = p.id where c.id = '{}'", container_id) + ], + )?; + + if pod_json.is_empty() { + serde_json::to_writer(io::stdout(), &json)?; + return Ok("ADD: not a pod".to_string()); + } + + let pod_value: Value = serde_json::from_str(&pod_json).map_err(|e| anyhow!("failed to parse pod json for {} from {}: {}", container_id, pod_json, e))?; + + let labels = pod_value["labels"].as_object().unwrap_or(&serde_json::Map::new()).clone(); + + if !labels.contains_key("app") || !labels.contains_key("skate.io/namespace") { + serde_json::to_writer(io::stdout(), &json)?; + return Ok("ADD: missing labels".to_string()); + } + + // domain is ..cluster.skate + let app = labels.get("app").ok_or(anyhow!("missing label"))?.as_str().unwrap_or_default().to_string(); + let ns = labels.get("skate.io/namespace").ok_or(anyhow!("missing label"))?.as_str().unwrap_or_default().to_string(); + if ns == "" { + serde_json::to_writer(io::stdout(), &json)?; + return Ok("ADD: namespace empty".to_string()); + } + let domain = format!("{}.{}.cluster.skate", app.clone(), ns.clone()); + let ip = result.ips[0].address.ip().to_string(); + + ensure_skatelet_cni_conf_dir(&config.name); + + // Do stuff + lock(&config.name, &|| { + let addnhosts_path = Path::new(&conf_path_str()).join(config.name.clone()).join("addnhosts"); + + // scope to make sure files closed after + { + + // create or open + let mut addhosts_file = OpenOptions::new() + .create(true) + .write(true) + .append(true) + .open(addnhosts_path).map_err(|e| anyhow!("failed to open addnhosts file: {}", e))?; + + writeln!(addhosts_file, "{} {} # {}", ip, domain, container_id).map_err(|e| anyhow!("failed to write host to file: {}", e))?; + } + + Ok(()) + })?; + + serde_json::to_writer(io::stdout(), &json)?; } "DEL" => { let json: Value = serde_json::from_reader(io::stdin()).map_err(|e| anyhow!("failed to parse stdin: {}", e))?; - if !json["prevResult"].is_object() { - return Err(anyhow!("failed to parse prevResult").into()); - } - - let prev_result = json["prevResult"].clone(); - let output = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; let config: NetworkConfig = serde_json::from_value(json.clone()).map_err(|e| anyhow!("failed to parse config: {}", e))?; + + let container_id = var("CNI_CONTAINERID")?; + // Do stuff - ensure_paths(&config.name); + ensure_skatelet_cni_conf_dir(&config.name); lock(&config.name, &|| { - let result = prev_result_or_default(&config); let _args = extract_args(&config); - let addnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts"); - let newaddnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts-new"); + let addnhosts_path = Path::new(&conf_path_str()).join(config.name.clone()).join("addnhosts"); + let newaddnhosts_path = Path::new(&conf_path_str()).join(config.name.clone()).join("addnhosts-new"); // scope to make sure files closed after { @@ -135,36 +211,32 @@ fn run() -> Result<(), Box> { let addhosts_file = OpenOptions::new() .read(true) - .open(addnhosts_path.clone()) - .unwrap(); + .open(addnhosts_path.clone()); + + if addhosts_file.is_err() { + return Ok(()); + } + let addhosts_file = addhosts_file?; let newaddhosts_file = OpenOptions::new() .create(true) .write(true) .truncate(true) - .open(newaddnhosts_path.clone()) - .unwrap(); + .open(newaddnhosts_path.clone())?; let reader = BufReader::new(&addhosts_file); let mut writer = BufWriter::new(&newaddhosts_file); - - if result.ips.len() == 0 { - return Err("no ips in prev_result".into()); - } - - let ip = result.ips[0].address.ip().to_string(); - for (_index, line) in reader.lines().enumerate() { - let line = line.as_ref().unwrap(); - if !line.starts_with(&ip) { + let line = line?; + if !line.ends_with(&container_id) { writeln!(writer, "{}", line)?; } } } fs::rename(&newaddnhosts_path, &addnhosts_path)?; - println!("{}", output); + serde_json::to_writer(io::stdout(), &json)?; Ok(()) })?; } @@ -174,8 +246,8 @@ fn run() -> Result<(), Box> { return Err(anyhow!("failed to parse prevResult").into()); } let prev_result = json["prevResult"].clone(); - let output = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; - print!("{}", output); + let response = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; + serde_json::to_writer(io::stdout(), &response).map_err(|e| anyhow!("failed to serialize version response: {}", e))?; } "VERSION" => { let json: Value = serde_json::from_reader(io::stdin()).map_err(|e| anyhow!("failed to parse stdin: {}", e))?; @@ -186,84 +258,14 @@ fn run() -> Result<(), Box> { let response = VersionReply { cni_version: cni_version.parse()?, - supported_versions: vec!["0.4.0".parse().unwrap()], + supported_versions: vec!["0.4.0".parse()?], }; serde_json::to_writer(io::stdout(), &response).map_err(|e| anyhow!("failed to serialize version response: {}", e))?; } _ => { - eprintln!("unknown command: {}", cmd); + return Err("unknown command".into()); } - - - // match Cni::load() { - // Cni::Del { container_id, ifname, netns, path, config } => { - // ensure_paths(&config.name); - // match lock(&config.name, &|| { - // let mut result = prev_result_or_default(&config); - // let args = extract_args(&config); - // - // let addnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts"); - // let newaddnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts-new"); - // - // // scope to make sure files closed after - // { - // // create or open - // - // let addhosts_file = OpenOptions::new() - // .read(true) - // .open(addnhosts_path.clone()) - // .unwrap(); - // - // let newaddhosts_file = OpenOptions::new() - // .create(true) - // .write(true) - // .truncate(true) - // .open(newaddnhosts_path.clone()) - // .unwrap(); - // - // let reader = BufReader::new(&addhosts_file); - // let mut writer = BufWriter::new(&newaddhosts_file); - // - // - // if result.ips.len() == 0 { - // return Err("no ips in prev_result".into()); - // } - // - // let ip = result.ips[0].address.ip().to_string(); - // - // for (index, line) in reader.lines().enumerate() { - // let line = line.as_ref().unwrap(); - // if !line.starts_with(&ip) { - // writeln!(writer, "{}", line)?; - // } - // } - // } - // fs::rename(&newaddnhosts_path, &addnhosts_path)?; - // Ok(()) - // }) { - // Err(e) => { - // reply(ErrorReply { - // cni_version: config.cni_version, - // code: 1, // TODO - // msg: &e.to_string(), - // details: "".to_string(), - // }) - // } - // Ok(()) => {} - // } - // reply(prev_result_or_default(&config)) - // } - // Cni::Check { container_id, ifname, netns, path, config } => { - // ensure_paths(&config.name); - // - // let prev_result = prev_result_or_default(&config); - // reply(prev_result); - // } - // Cni::Version(_) => { - // eprintln!("version"); - // } - // } }; - Ok(()) + Ok(cmd) } \ No newline at end of file diff --git a/src/skatelet/mod.rs b/src/skatelet/mod.rs index 9c64cb2..d6f7164 100644 --- a/src/skatelet/mod.rs +++ b/src/skatelet/mod.rs @@ -3,7 +3,6 @@ mod apply; mod system; mod cni; -mod ocihooks; pub use skatelet::skatelet; pub use system::SystemInfo; diff --git a/src/skatelet/ocihooks.rs b/src/skatelet/ocihooks.rs deleted file mode 100644 index 6d9d9b5..0000000 --- a/src/skatelet/ocihooks.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::collections::HashMap; -use std::env; -use std::error::Error; -use std::fs::{File, OpenOptions}; -use std::path::{Path, PathBuf}; -use anyhow::anyhow; -use clap::{Args, Subcommand}; -use fs2::FileExt; -use serde::{Deserialize, Serialize}; -use crate::skatelet::skatelet::VAR_PATH; - -#[derive(Debug, Args)] -pub struct HookArgs { - #[command(subcommand)] - command: HookCommand, -} - - -#[derive(Debug, Subcommand)] -pub enum HookCommand { - #[command(name = "prestart", about = "prestart hook")] - Prestart, - #[command(name = "poststop", about = "poststop hook")] - Poststop, -} - -pub fn oci(apply_args: HookArgs) -> Result<(), Box> { - match apply_args.command { - HookCommand::Prestart => pre_start(), - HookCommand::Poststop => post_stop() - } -} - -#[derive(Serialize, Deserialize)] -struct config { - annotations: HashMap, -} - -fn lock(lock_path: &PathBuf, cb: &dyn Fn() -> Result>) -> Result> { - let lock_file = File::create(lock_path.clone()).map_err(|e| anyhow!("failed to create/open lock file {}: {}",lock_path.to_string_lossy(), e))?; - lock_file.lock_exclusive()?; - let result = cb(); - lock_file.unlock()?; - result -} - -fn pre_start() -> Result<(), Box> { - let config_file = File::open("./config.json").map_err(|e| anyhow!("failed to open config.json: {}", e))?; - let conf: config = serde_json::from_reader(config_file).map_err(|e| anyhow!("failed to read config.json: {}", e))?; - let ns = conf.annotations.get("skate.io/namespace"); - - if ns.is_none() { - return Ok(()); - } - - let _ns = ns.unwrap(); - - - let cwd = env::current_dir().map_err(|e| anyhow!("failed to get cwd: {}", e))?; - let _container_id = cwd.parent().unwrap().file_name().unwrap().to_str().unwrap(); - - let dns_path = format!("{}/dns", VAR_PATH); - - lock(&Path::new(&dns_path).join("lock"), &|| { - let addnhosts_path = Path::new(&dns_path).join("addnhosts"); - // create or open - let _addhosts_file = OpenOptions::new() - .create(true) - .write(true) - .append(true) - .open(addnhosts_path).map_err(|e| anyhow!("failed to open addnhosts file: {}", e))?; - - let _names: Vec = vec![]; - - - // if result.ips.len() == 0 { - // return Err("no ips in prev_result".into()); - // } - // - // let ip_str = result.ips[0].address.ip().to_string(); - // - // for name in names { - // writeln!(addhosts_file, "{} {}", ip_str, name).map_err(|e| anyhow!("failed to write host to file: {}", e))?; - // } - Ok(()) - })?; - - // // write to /var/lib/skatelet/pods//ns/ - // let dir = format!("{}/containers/{}", VAR_PATH, container_id); - // create_dir_all(dir.clone()).map_err(|e| anyhow!("failed to create container dir: {}", e))?; - // - // let mut file = File::create(format!("{}/ns", dir)).map_err(|e| anyhow!("failed to create container ns file: {}", e))?; - // file.write(ns.as_bytes()).map_err(|e| anyhow!("failed to write container ns file: {}", e))?; - Ok(()) -} - -fn list_sub_dirs(path: &str) -> Vec { - let dir = std::fs::read_dir(path).map_err(|e| anyhow!("failed to read container dir: {}", e)); - if dir.is_err() { - return vec![]; - } - let dir = dir.unwrap(); - - return dir.filter_map(|d| { - if let Ok(d) = d { - if let Ok(t) = d.file_type() { - if t.is_dir() { - return Some(d.file_name().to_string_lossy().to_string()); - } - } - } - return None; - }).collect(); -} - -fn post_stop() -> Result<(), Box> { - // let cwd = env::current_dir().map_err(|e| anyhow!("failed to get cwd: {}", e))?; - // - // let skate_containers = list_sub_dirs(format!("{}/containers", VAR_PATH).as_str()); - // // TODO - use podman cli - // let podman_containers = list_sub_dirs("/var/lib/containers/storage/overlay-containers"); - // - // println!("skate_containers: {:?}", skate_containers); - // println!("podman_containers: {:?}", podman_containers); - // - // for skate_container in skate_containers { - // if !podman_containers.contains(&skate_container) { - // let dir = format!("{}/containers/{}", VAR_PATH, skate_container); - // remove_dir_all(dir.clone()).map_err(|e| anyhow!("failed to remove container dir {}: {}",dir, e))?; - // } - // } - // - Ok(()) -} diff --git a/src/skatelet/skatelet.rs b/src/skatelet/skatelet.rs index 84568ac..d63bf2f 100644 --- a/src/skatelet/skatelet.rs +++ b/src/skatelet/skatelet.rs @@ -3,7 +3,6 @@ use clap::{Parser, Subcommand}; use crate::skatelet::apply; use crate::skatelet::apply::{ApplyArgs, remove, RemoveArgs}; use crate::skatelet::cni::cni; -use crate::skatelet::ocihooks::{HookArgs, oci}; use crate::skatelet::system::{system, SystemArgs}; pub const VAR_PATH: &str = "/var/lib/skatelet"; @@ -22,7 +21,6 @@ enum Commands { System(SystemArgs), Remove(RemoveArgs), Cni, - Oci(HookArgs) } pub async fn skatelet() -> Result<(), Box> { @@ -35,7 +33,6 @@ pub async fn skatelet() -> Result<(), Box> { cni(); Ok(()) }, - Commands::Oci(args) => oci(args) // _ => Ok(()) } } diff --git a/src/state/state.rs b/src/state/state.rs index 0ee3988..58027e7 100644 --- a/src/state/state.rs +++ b/src/state/state.rs @@ -96,6 +96,7 @@ impl Into for NodeState { Some(BTreeMap::::from([ ("skate.io/arch".to_string(), si.platform.arch.clone()), ("skate.io/os".to_string(), si.platform.os.to_string().to_lowercase()), + ("skate.io/nodename".to_string(), self.node_name.clone()), ("skate.io/hostname".to_string(), si.hostname.clone()), ])) )) @@ -138,10 +139,22 @@ impl ClusterState { } pub fn load(cluster_name: &str) -> Result> { - let file = File::open(ClusterState::path(cluster_name))?; - - let result: ClusterState = serde_json::from_reader(file)?; - Ok(result) + let file = File::create(ClusterState::path(cluster_name)).map_err(|e| anyhow!("failed to open or create state file").context(e))?; + + let result = serde_json::from_reader::<_, ClusterState>(file).map_err(|e| anyhow!("failed to parse cluster state").context(e)); + + match result { + Ok(state) => Ok(state), + Err(e) => { + let state = ClusterState { + cluster_name: cluster_name.to_string(), + hash: "".to_string(), + nodes: vec![], + }; + state.persist()?; + Ok(state) + } + } } pub fn reconcile_node(&mut self, node: &NodeSystemInfo) -> Result> {