From 6a616c35f39fb6b779dad2109f2756fd81d31d76 Mon Sep 17 00:00:00 2001 From: Donal Byrne Date: Thu, 11 Jul 2024 23:10:17 +0200 Subject: [PATCH] WIP --- hack/test-deployment.yaml | 8 +- manifests/coredns.yaml | 12 +- src/create.rs | 24 +++- src/executor.rs | 2 +- src/skatelet/cni.rs | 232 +++++++++++++++++++------------------- src/skatelet/ocihooks.rs | 2 +- src/state/state.rs | 20 +++- 7 files changed, 166 insertions(+), 134 deletions(-) diff --git a/hack/test-deployment.yaml b/hack/test-deployment.yaml index 624b5b0..831f778 100644 --- a/hack/test-deployment.yaml +++ b/hack/test-deployment.yaml @@ -7,7 +7,7 @@ metadata: labels: app: nginx spec: - replicas: 0 + replicas: 1 selector: matchLabels: app: nginx @@ -18,8 +18,8 @@ spec: app: nginx spec: containers: - - name: nginx + - name: nginx1 + image: nginx:1.14.2 + - name: nginx2 image: nginx:1.14.2 -# ports: -# - containerPort: 8000 --- diff --git a/manifests/coredns.yaml b/manifests/coredns.yaml index bff1f50..981d1e1 100644 --- a/manifests/coredns.yaml +++ b/manifests/coredns.yaml @@ -17,15 +17,15 @@ spec: spec: hostNetwork: true volumes: - - name: skate + - name: cni hostPath: - path: /etc/skate + path: /var/lib/skatelet/cni/podman containers: - name: coredns image: ghcr.io/skateco/coredns volumeMounts: - - mountPath: /etc/skate - name: skate + - mountPath: /var/lib/skatelet/cni/podman + name: cni env: - name: CORE_FILE value: | @@ -33,7 +33,7 @@ spec: bind lo 0.0.0.0 - hosts /run/containers/cni/dnsname/podman/addnhosts + hosts /var/lib/skatelet/cni/podman/addnhosts } cluster.skate:53 { @@ -45,7 +45,7 @@ spec: loadbalance round_robin } - . { + .:53 { bind lo 0.0.0.0 forward . 8.8.8.8 cache diff --git a/src/create.rs b/src/create.rs index c725e00..f6c6de8 100644 --- a/src/create.rs +++ b/src/create.rs @@ -183,11 +183,16 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box> { } async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, _info: &NodeSystemInfo, args: &CreateNodeArgs) -> Result<(), Box> { + let cmd = "sqlite3 -version || sudo apt-get install -y sqlite3"; + conn.execute(cmd).await?; + let cmd = "sudo cp /usr/share/containers/containers.conf /etc/containers/containers.conf"; conn.execute(cmd).await?; let cmd = format!("sudo sed -i 's&#default_subnet[ =].*&default_subnet = \"{}\"&' /etc/containers/containers.conf", node.subnet_cidr); conn.execute(&cmd).await?; + let cmd = "sudo sed -i 's&#network_backend[ =].*&network_backend = \"cni\"&' /etc/containers/containers.conf"; + conn.execute(&cmd).await?; let cmd = "sudo ip link del cni-podman0|| exit 0"; conn.execute(&cmd).await?; @@ -243,10 +248,18 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, // In ubuntu 24.04 there's an issue with apparmor and podman // https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2040483 - let cmd = "sudo systemctl disable apparmor.service --now"; - conn.execute(cmd).await?; + + let cmd = "sudo systemctl list-unit-files apparmor.service"; + let apparmor_unit_exists = conn.execute(cmd).await; + + if apparmor_unit_exists.is_ok() { + let cmd = "sudo systemctl disable apparmor.service --now"; + conn.execute(cmd).await?; + } let cmd = "sudo aa-teardown"; _ = conn.execute(cmd).await; + let cmd = "sudo apt purge -y apparmor"; + _ = conn.execute(cmd).await; // // install dnsmasq @@ -257,12 +270,17 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node, conn.execute(cmd).await?; // changed /etc/resolv.conf to be 127.0.0.1 let cmd = "sudo bash -c 'echo 127.0.0.1 > /etc/resolv.conf'"; - conn.execute(cmd).await?; + _ = conn.execute(cmd).await; /// COREDNS /// coredns listens on port 53 and 5533 /// port 53 serves .cluster.skate by forwarding to all coredns instances on port 5553 /// uses fanout plugin + + // needed for mount + let cmd = "sudo mkdir -p /run/containers/cni"; + conn.execute(cmd).await?; + let coredns_yaml_path = "/tmp/skate-coredns.yaml"; let mut file = File::create(coredns_yaml_path)?; // replace forward list in coredns config with that of other hosts diff --git a/src/executor.rs b/src/executor.rs index 4b80315..385bcdd 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -31,7 +31,7 @@ impl Executor for DefaultExecutor { let file_path = DefaultExecutor::write_to_file(&serde_yaml::to_string(&object)?)?; - let args = ["play", "kube", &file_path, "--start"]; + let args = ["play", "kube", &file_path, "--start", "--network=podman" ]; let output = process::Command::new("podman") .args(args) .stdin(Stdio::piped()) diff --git a/src/skatelet/cni.rs b/src/skatelet/cni.rs index 59cdf02..a81a84e 100644 --- a/src/skatelet/cni.rs +++ b/src/skatelet/cni.rs @@ -14,15 +14,15 @@ use anyhow::anyhow; use cni_plugin::config::NetworkConfig; use serde_json::Value; use serde_json::Value::String as JsonString; +use crate::skate::exec_cmd; - -fn conf_path() -> String { +fn conf_path_str() -> String { "/var/lib/skatelet/cni".to_string() } fn lock(network_name: &str, cb: &dyn Fn() -> Result>) -> Result> { - let lock_path = Path::new(&conf_path()).join(network_name).join("lock"); + let lock_path = Path::new(&conf_path_str()).join(network_name).join("lock"); let lock_file = File::create(lock_path.clone()).map_err(|e| anyhow!("failed to create/open lock file: {}", e))?; debug!("waiting for lock on {}", lock_path.display()); lock_file.lock_exclusive()?; @@ -35,10 +35,10 @@ fn lock(network_name: &str, cb: &dyn Fn() -> Result>) -> Re result } -fn ensure_paths(net_name: &str) { - let conf_path_str = conf_path(); - let conf_path = Path::new(&conf_path_str); - let net_path = conf_path.join(net_name); +fn ensure_skatelet_cni_conf_dir(dir_name: &str) { + let conf_str = conf_path_str(); + let conf_path = Path::new(&conf_str); + let net_path = conf_path.join(dir_name); fs::create_dir_all(conf_path).unwrap(); fs::create_dir_all(net_path).unwrap(); @@ -89,45 +89,121 @@ fn extract_prev_result(prev_value: Option) -> Option { }) } +fn write_last_run_file(msg: &str) { + let last_err_path = Path::new(&conf_path_str()).join(".last_run.log"); + let mut last_err_file = OpenOptions::new() + .create(true) + .write(true) + .append(true) + .open(last_err_path).unwrap(); + writeln!(last_err_file, "{}", msg).unwrap(); +} + pub fn cni() { match run() { - Ok(_) => {} - Err(_e) => { - // handle error formatting + Ok(warning) => { + write_last_run_file(&format!("WARNING: {}", warning)) + } + Err(e) => { + write_last_run_file(&format!("ERROR: {}", e)); + panic!("{}", e) } } } -fn run() -> Result<(), Box> { +fn run() -> Result> { + let conf_str = conf_path_str(); + let conf_path = Path::new(&conf_str); + + fs::create_dir_all(conf_path).unwrap(); + let cmd = var("CNI_COMMAND").unwrap_or_default(); match cmd.as_str() { "ADD" => { let json: Value = serde_json::from_reader(io::stdin()).map_err(|e| anyhow!("failed to parse stdin: {}", e))?; - if !json["prevResult"].is_object() { - return Err(anyhow!("failed to parse prevResult").into()); + + let config: NetworkConfig = serde_json::from_value(json.clone()).map_err(|e| anyhow!("failed to parse config: {}", e))?; + + let result = prev_result_or_default(&config); + + if result.ips.len() == 0 { + return Err("no ips in prev_result".into()); } - let prev_result = json["prevResult"].clone(); - let output = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; - print!("{}", output); + + + let container_id = var("CNI_CONTAINERID")?; + + // get podman info from sqlitedb in /var/lib/containers/storage/db.sql + let pod_json = exec_cmd( + "sqlite3", + &[ + "/var/lib/containers/storage/db.sql", + &format!("select p.json from ContainerConfig c join PodConfig p on c.PodID = p.id where c.id = '{}'", container_id) + ], + )?; + + if pod_json.is_empty() { + serde_json::to_writer(io::stdout(), &json)?; + return Ok("ADD: not a pod".to_string()); + } + + let pod_value: Value = serde_json::from_str(&pod_json).map_err(|e| anyhow!("failed to parse pod json for {} from {}: {}", container_id, pod_json, e))?; + + let labels = pod_value["labels"].as_object().unwrap_or(&serde_json::Map::new()).clone(); + + if !labels.contains_key("app") || !labels.contains_key("skate.io/namespace") { + serde_json::to_writer(io::stdout(), &json)?; + return Ok("ADD: missing labels".to_string()); + } + + // domain is ..cluster.skate + let app = labels.get("app").ok_or(anyhow!("missing label"))?.as_str().unwrap_or_default().to_string(); + let ns = labels.get("skate.io/namespace").ok_or(anyhow!("missing label"))?.as_str().unwrap_or_default().to_string(); + if ns == "" { + serde_json::to_writer(io::stdout(), &json)?; + return Ok("ADD: namespace empty".to_string()); + } + let domain = format!("{}.{}.cluster.skate", app.clone(), ns.clone()); + let ip = result.ips[0].address.ip().to_string(); + + ensure_skatelet_cni_conf_dir(&config.name); + + // Do stuff + lock(&config.name, &|| { + let addnhosts_path = Path::new(&conf_path_str()).join(config.name.clone()).join("addnhosts"); + + // scope to make sure files closed after + { + + // create or open + let mut addhosts_file = OpenOptions::new() + .create(true) + .write(true) + .append(true) + .open(addnhosts_path).map_err(|e| anyhow!("failed to open addnhosts file: {}", e))?; + + writeln!(addhosts_file, "{} {} # {}", ip, domain, container_id).map_err(|e| anyhow!("failed to write host to file: {}", e))?; + } + + Ok(()) + })?; + + serde_json::to_writer(io::stdout(), &json)?; } "DEL" => { let json: Value = serde_json::from_reader(io::stdin()).map_err(|e| anyhow!("failed to parse stdin: {}", e))?; - if !json["prevResult"].is_object() { - return Err(anyhow!("failed to parse prevResult").into()); - } - - let prev_result = json["prevResult"].clone(); - let output = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; let config: NetworkConfig = serde_json::from_value(json.clone()).map_err(|e| anyhow!("failed to parse config: {}", e))?; + + let container_id = var("CNI_CONTAINERID")?; + // Do stuff - ensure_paths(&config.name); + ensure_skatelet_cni_conf_dir(&config.name); lock(&config.name, &|| { - let result = prev_result_or_default(&config); let _args = extract_args(&config); - let addnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts"); - let newaddnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts-new"); + let addnhosts_path = Path::new(&conf_path_str()).join(config.name.clone()).join("addnhosts"); + let newaddnhosts_path = Path::new(&conf_path_str()).join(config.name.clone()).join("addnhosts-new"); // scope to make sure files closed after { @@ -135,36 +211,32 @@ fn run() -> Result<(), Box> { let addhosts_file = OpenOptions::new() .read(true) - .open(addnhosts_path.clone()) - .unwrap(); + .open(addnhosts_path.clone()); + + if addhosts_file.is_err() { + return Ok(()); + } + let addhosts_file = addhosts_file?; let newaddhosts_file = OpenOptions::new() .create(true) .write(true) .truncate(true) - .open(newaddnhosts_path.clone()) - .unwrap(); + .open(newaddnhosts_path.clone())?; let reader = BufReader::new(&addhosts_file); let mut writer = BufWriter::new(&newaddhosts_file); - - if result.ips.len() == 0 { - return Err("no ips in prev_result".into()); - } - - let ip = result.ips[0].address.ip().to_string(); - for (_index, line) in reader.lines().enumerate() { - let line = line.as_ref().unwrap(); - if !line.starts_with(&ip) { + let line = line?; + if !line.ends_with(&container_id) { writeln!(writer, "{}", line)?; } } } fs::rename(&newaddnhosts_path, &addnhosts_path)?; - println!("{}", output); + serde_json::to_writer(io::stdout(), &json)?; Ok(()) })?; } @@ -174,8 +246,8 @@ fn run() -> Result<(), Box> { return Err(anyhow!("failed to parse prevResult").into()); } let prev_result = json["prevResult"].clone(); - let output = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; - print!("{}", output); + let response = serde_json::to_string(&prev_result).map_err(|e| anyhow!("failed to serialize prev result: {}", e))?; + serde_json::to_writer(io::stdout(), &response).map_err(|e| anyhow!("failed to serialize version response: {}", e))?; } "VERSION" => { let json: Value = serde_json::from_reader(io::stdin()).map_err(|e| anyhow!("failed to parse stdin: {}", e))?; @@ -186,84 +258,14 @@ fn run() -> Result<(), Box> { let response = VersionReply { cni_version: cni_version.parse()?, - supported_versions: vec!["0.4.0".parse().unwrap()], + supported_versions: vec!["0.4.0".parse()?], }; serde_json::to_writer(io::stdout(), &response).map_err(|e| anyhow!("failed to serialize version response: {}", e))?; } _ => { - eprintln!("unknown command: {}", cmd); + return Err("unknown command".into()); } - - - // match Cni::load() { - // Cni::Del { container_id, ifname, netns, path, config } => { - // ensure_paths(&config.name); - // match lock(&config.name, &|| { - // let mut result = prev_result_or_default(&config); - // let args = extract_args(&config); - // - // let addnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts"); - // let newaddnhosts_path = Path::new(&conf_path()).join(config.name.clone()).join("addnhosts-new"); - // - // // scope to make sure files closed after - // { - // // create or open - // - // let addhosts_file = OpenOptions::new() - // .read(true) - // .open(addnhosts_path.clone()) - // .unwrap(); - // - // let newaddhosts_file = OpenOptions::new() - // .create(true) - // .write(true) - // .truncate(true) - // .open(newaddnhosts_path.clone()) - // .unwrap(); - // - // let reader = BufReader::new(&addhosts_file); - // let mut writer = BufWriter::new(&newaddhosts_file); - // - // - // if result.ips.len() == 0 { - // return Err("no ips in prev_result".into()); - // } - // - // let ip = result.ips[0].address.ip().to_string(); - // - // for (index, line) in reader.lines().enumerate() { - // let line = line.as_ref().unwrap(); - // if !line.starts_with(&ip) { - // writeln!(writer, "{}", line)?; - // } - // } - // } - // fs::rename(&newaddnhosts_path, &addnhosts_path)?; - // Ok(()) - // }) { - // Err(e) => { - // reply(ErrorReply { - // cni_version: config.cni_version, - // code: 1, // TODO - // msg: &e.to_string(), - // details: "".to_string(), - // }) - // } - // Ok(()) => {} - // } - // reply(prev_result_or_default(&config)) - // } - // Cni::Check { container_id, ifname, netns, path, config } => { - // ensure_paths(&config.name); - // - // let prev_result = prev_result_or_default(&config); - // reply(prev_result); - // } - // Cni::Version(_) => { - // eprintln!("version"); - // } - // } }; - Ok(()) + Ok(cmd) } \ No newline at end of file diff --git a/src/skatelet/ocihooks.rs b/src/skatelet/ocihooks.rs index 6d9d9b5..6cb4b54 100644 --- a/src/skatelet/ocihooks.rs +++ b/src/skatelet/ocihooks.rs @@ -76,7 +76,7 @@ fn pre_start() -> Result<(), Box> { // if result.ips.len() == 0 { // return Err("no ips in prev_result".into()); // } - // + // let ip_str = result.ips[0].address.ip().to_string(); // // for name in names { diff --git a/src/state/state.rs b/src/state/state.rs index 0ee3988..29a688d 100644 --- a/src/state/state.rs +++ b/src/state/state.rs @@ -138,10 +138,22 @@ impl ClusterState { } pub fn load(cluster_name: &str) -> Result> { - let file = File::open(ClusterState::path(cluster_name))?; - - let result: ClusterState = serde_json::from_reader(file)?; - Ok(result) + let file = File::create(ClusterState::path(cluster_name)).map_err(|e| anyhow!("failed to open or create state file").context(e))?; + + let result = serde_json::from_reader::<_, ClusterState>(file).map_err(|e| anyhow!("failed to parse cluster state").context(e)); + + match result { + Ok(state) => Ok(state), + Err(e) => { + let state = ClusterState { + cluster_name: cluster_name.to_string(), + hash: "".to_string(), + nodes: vec![], + }; + state.persist()?; + Ok(state) + } + } } pub fn reconcile_node(&mut self, node: &NodeSystemInfo) -> Result> {