From edfac138726f2adf0992f3c4983eda8d74590be4 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Tue, 20 Aug 2024 16:58:36 +0200
Subject: [PATCH 01/32] add vm-network-utils

---
 Cargo.lock                          |  44 +++-
 Cargo.toml                          |   1 +
 crates/vm-network-utils/Cargo.toml  |  11 +
 crates/vm-network-utils/src/lib.rs  | 322 ++++++++++++++++++++++++++++
 crates/vm-network-utils/src/main.rs |  16 ++
 5 files changed, 391 insertions(+), 3 deletions(-)
 create mode 100644 crates/vm-network-utils/Cargo.toml
 create mode 100644 crates/vm-network-utils/src/lib.rs
 create mode 100644 crates/vm-network-utils/src/main.rs

diff --git a/Cargo.lock b/Cargo.lock
index fd904138bb..9009f9e024 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1395,6 +1395,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
 
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
 [[package]]
 name = "chacha20"
 version = "0.9.1"
@@ -3816,6 +3822,17 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
 
+[[package]]
+name = "iptables"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43b627935a2f5d654613bea2bcd677cc760b03ecf224ced0f1970c0d174813b9"
+dependencies = [
+ "lazy_static",
+ "nix 0.29.0",
+ "regex",
+]
+
 [[package]]
 name = "is-terminal"
 version = "0.4.9"
@@ -4249,9 +4266,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
 
 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libipld"
@@ -5730,11 +5747,23 @@ checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
 dependencies = [
  "bitflags 2.4.1",
  "cfg-if",
- "cfg_aliases",
+ "cfg_aliases 0.1.1",
  "libc",
  "memoffset",
 ]
 
+[[package]]
+name = "nix"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
+dependencies = [
+ "bitflags 2.4.1",
+ "cfg-if",
+ "cfg_aliases 0.2.1",
+ "libc",
+]
+
 [[package]]
 name = "nohash-hasher"
 version = "0.2.0"
@@ -9184,6 +9213,15 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "vm-network-utils"
+version = "0.1.0"
+dependencies = [
+ "iptables",
+ "thiserror",
+ "tracing",
+]
+
 [[package]]
 name = "vm-utils"
 version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 59a6f5a86f..2c288ec526 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -49,6 +49,7 @@ members = [
     "crates/core-distributor",
     "crates/log-format",
     "crates/vm-utils",
+    "crates/vm-network-utils",
 ]
 exclude = [
     "nox/tests/tetraplets",
diff --git a/crates/vm-network-utils/Cargo.toml b/crates/vm-network-utils/Cargo.toml
new file mode 100644
index 0000000000..38231f476d
--- /dev/null
+++ b/crates/vm-network-utils/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "vm-network-utils"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dependencies]
+iptables = "0.5.2"
+
+thiserror = { workspace = true }
+tracing = { workspace = true }
\ No newline at end of file
diff --git a/crates/vm-network-utils/src/lib.rs b/crates/vm-network-utils/src/lib.rs
new file mode 100644
index 0000000000..880fd841cf
--- /dev/null
+++ b/crates/vm-network-utils/src/lib.rs
@@ -0,0 +1,322 @@
+use iptables::IPTables;
+use thiserror::Error;
+
+pub struct NetworkSettings {
+    pub public_ip: String,
+    pub vm_ip: String,
+    pub bridge_name: String,
+    pub port_range: (u16, u16),
+    pub host_ssh_port: u16,
+    pub vm_ssh_port: u16,
+}
+
+#[derive(Debug, Error)]
+#[error(transparent)]
+pub struct DNatSetupError(#[from] Box<dyn std::error::Error>);
+
+#[derive(Debug, Error)]
+#[error(transparent)]
+pub struct SNatSetupError(#[from] Box<dyn std::error::Error>);
+
+#[derive(Debug, Error)]
+#[error(transparent)]
+pub struct FwdSetupError(#[from] Box<dyn std::error::Error>);
+
+#[derive(Debug, Error)]
+pub enum NetworkSetupError {
+    #[error("error setting up DNAT rules: {0}")]
+    DNat(#[from] DNatSetupError),
+    #[error("error setting up SNAT rules: {0}")]
+    SNat(#[from] SNatSetupError),
+    #[error("error setting up FWD rules: {0}")]
+    Fwd(#[from] FwdSetupError),
+    #[error("error initializing iptables: {0}")]
+    Init(#[from] Box<dyn std::error::Error>),
+}
+
+pub fn setup_network(
+    network_settings: NetworkSettings,
+    name: &str,
+) -> Result<(), NetworkSetupError> {
+    let ipt = iptables::new(false)?;
+    setup_snat(&network_settings, &ipt, name)?;
+    setup_dnat(&network_settings, &ipt, name)?;
+    setup_fwd(&network_settings, &ipt, name)?;
+    Ok(())
+}
+
+pub fn clear_network(
+    network_settings: &NetworkSettings,
+    name: &str,
+) -> Result<(), NetworkSetupError> {
+    let ipt = iptables::new(false)?;
+    let snat_result = {
+        let (chain_name, rules) = snat_rules(network_settings, name);
+        clear_chain(&ipt, &chain_name, &rules)
+    };
+
+    let dnat_result = {
+        let (chain_name, rules) = dnat_rules(network_settings, name);
+        clear_chain(&ipt, &chain_name, &rules)
+    };
+
+    let fwd_result = {
+        let (chain_name, rules) = fwd_rules(network_settings, name);
+        clear_chain(&ipt, &chain_name, &rules)
+    };
+
+    // Try to clean as much as possible and only then fail
+    snat_result?;
+    dnat_result?;
+    fwd_result?;
+
+    Ok(())
+}
+
+fn clear_chain(
+    ipt: &IPTables,
+    chain_name: &str,
+    rules: &Vec<String>,
+) -> Result<(), NetworkSetupError> {
+    let exists = ipt.chain_exists("nat", chain_name)?;
+    if !exists {
+        tracing::warn!("Can't clean the chain {chain_name}: doesn't exist");
+        return Ok(());
+    }
+
+    for rule in rules {
+        if ipt.exists("nat", chain_name, rule)? {
+            ipt.delete("nat", chain_name, rule)?;
+        } else {
+            tracing::warn!(
+                "Can't clean a rule for the chain {chain_name}: doesn't exist; rule: {rule}"
+            )
+        }
+    }
+
+    ipt.delete_chain("nat", chain_name)?;
+    Ok(())
+}
+
+fn setup_dnat(
+    network_settings: &NetworkSettings,
+    ipt: &IPTables,
+    name: &str,
+) -> Result<(), DNatSetupError> {
+    let (chain_name, rules) = dnat_rules(network_settings, name);
+    apply_rules(ipt, &chain_name, &rules)?;
+    Ok(())
+}
+
+fn setup_snat(
+    network_settings: &NetworkSettings,
+    ipt: &IPTables,
+    name: &str,
+) -> Result<(), SNatSetupError> {
+    let (chain_name, rules) = snat_rules(network_settings, name);
+    apply_rules(ipt, &chain_name, &rules)?;
+    Ok(())
+}
+
+fn setup_fwd(
+    network_settings: &NetworkSettings,
+    ipt: &IPTables,
+    name: &str,
+) -> Result<(), FwdSetupError> {
+    let (chain_name, rules) = fwd_rules(network_settings, name);
+    apply_rules(ipt, &chain_name, &rules)?;
+    Ok(())
+}
+
+fn apply_rules(
+    ipt: &IPTables,
+    name: &str,
+    rules: &Vec<String>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    ipt.new_chain("nat", &name)?;
+    for rule in rules {
+        ipt.append("nat", &name, &rule)?;
+    }
+    Ok(())
+}
+
+// ```
+// iptables -t nat -N SNAT-${VM_NAME}
+// # Map the port range
+// iptables -A SNAT-${VM_NAME}
+//          -s ${VM_IP}
+//          -p tcp -m tcp
+//           --dport ${RNG_START}:${RNG_END}
+//          -j SNAT
+//          --to-source ${PUBLIC_IP}
+// iptables -A SNAT-${VM_NAME}
+//          -s ${VM_IP}
+//          -p tcp -m tcp
+//          --dport ${RNG_START}:${RNG_END}
+//          -d ${PUBLIC_IP}
+//          -j MASQUERADE
+// # Map SSH ports
+// iptables -A SNAT-${VM_NAME}
+//          -s ${VM_IP}
+//          -p tcp -m tcp
+//          --dport ${MAP_VM}
+//          -j SNAT
+//          --to-source ${PUBLIC_IP}
+// iptables -A SNAT-${VM_NAME}
+//          -s ${VM_IP}
+//          -p tcp -m tcp
+//          --dport ${MAP_HOST}
+//          -d ${PUBLIC_IP}
+//          -j MASQUERADE
+// ```
+fn snat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+    let name = format!("SNAT-{name}");
+    let common_prefix = &[
+        format!("-s {}", network_settings.vm_ip),
+        "-p tcp".to_string(),
+        "-m tcp".to_string(),
+    ]
+    .join(" ");
+
+    let ports_rules = &[
+        common_prefix.clone(),
+        format!(
+            "--dport {}:{}",
+            network_settings.port_range.0, network_settings.port_range.1
+        ),
+        "-j SNAT".to_string(),
+        format!("--to-source {}", network_settings.public_ip),
+    ]
+    .join(" ");
+
+    let ssh_ports_rules = &[
+        common_prefix.clone(),
+        format!("--dport {}", network_settings.vm_ssh_port),
+        "-j SNAT".to_string(),
+        format!("--to-source {}", network_settings.public_ip),
+    ]
+    .join(" ");
+
+    let masquerade_ports_rules = &[
+        common_prefix.clone(),
+        format!(
+            "--dport {}:{}",
+            network_settings.port_range.0, network_settings.port_range.1
+        ),
+        format!("-d {}", network_settings.public_ip),
+        "-j MASQUERADE".to_string(),
+    ]
+    .join(" ");
+    let masquerade_ssh_ports_rules = &[
+        common_prefix.clone(),
+        format!("--dport {}", network_settings.host_ssh_port),
+        format!("-d {}", network_settings.public_ip),
+        "-j MASQUERADE".to_string(),
+    ]
+    .join(" ");
+
+    let rules = vec![
+        ports_rules.to_string(),
+        ssh_ports_rules.to_string(),
+        masquerade_ports_rules.to_string(),
+        masquerade_ssh_ports_rules.to_string(),
+    ];
+
+    (name, rules)
+}
+
+// The corresponding iptables commands:
+// ```bash
+// iptables -t nat -N DNAT-${VM_NAME}
+// # Map the port range
+// iptables -A DNAT-${VM_NAME}  # --append chain
+//          -d ${PUBLIC_IP}     # --destination
+//          -p tcp              # --protocol
+//          -m tcp              # --match
+//          --dport ${RNG_START}:${RNG_END} # --destination-port (I don't have it my manual)
+//          --to-destination ${VM_IP}:${RNG_START}-${RNG_END}
+//          -j DNAT             # --jump
+// # Map the SSH ports
+// iptables -A DNAT-${VM_NAME}
+//          -d ${PUBLIC_IP}
+//          -p tcp -m tcp
+//          --dport ${MAP_HOST}
+//           --to-destination ${VM_IP}:${MAP_VM}
+//           -j DNAT
+// ```
+fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+    let name = format!("DNAT-{name}");
+    let common_prefix = &[
+        format!("-d {}", network_settings.public_ip),
+        "-p tcp".to_string(),
+        "-m tcp".to_string(),
+        format!(
+            "--dport {}:{}",
+            network_settings.port_range.0, network_settings.port_range.1
+        ),
+        "-j DNAT".to_string(),
+    ]
+    .join(" ");
+
+    let port_rules = &[
+        common_prefix.clone(),
+        format!(
+            "--to-destination {}:{}-{}",
+            network_settings.vm_ip, network_settings.port_range.0, network_settings.port_range.1
+        ),
+    ]
+    .join(" ");
+    let ssh_port_rules = &[
+        common_prefix.clone(),
+        format!(
+            "--to-destination {}:{}",
+            network_settings.vm_ip, network_settings.vm_ssh_port
+        ),
+    ]
+    .join(" ");
+    let rules = vec![port_rules.to_string(), ssh_port_rules.to_string()];
+    (name, rules)
+}
+
+// ```
+// iptables -t nat -N FWD-${VM_NAME}
+// iptables -A FWD-${VM_NAME}
+//          -d ${VM_IP}
+//          -o ${BRIDGE_NAME}   # --out-interface
+//          -p tcp -m tcp
+//          --dport ${RNG_START}:${RNG_END}
+//          -j ACCEPT
+// iptables -A FWD-${VM_NAME}
+//          -d ${VM_IP}
+//          -o ${BRIDGE_NAME}
+//          -p tcp -m tcp
+//          --dport ${MAP_VM}
+//          -j ACCEPT
+// ```
+fn fwd_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+    let name = format!("FWD-{name}");
+    let common_prefix = &[
+        format!("-d {}", network_settings.public_ip),
+        "-p tcp".to_string(),
+        "-m tcp".to_string(),
+        format!("-o {}", network_settings.bridge_name),
+    ]
+    .join(" ");
+    let port_rules = &[
+        common_prefix.clone(),
+        format!(
+            "--dport {}:{}",
+            network_settings.port_range.0, network_settings.port_range.1
+        ),
+        "-j ACCEPT".to_string(),
+    ]
+    .join(" ");
+    let ssh_rules = &[
+        common_prefix.clone(),
+        format!("--dport {}", network_settings.vm_ssh_port),
+        "-j ACCEPT".to_string(),
+    ]
+    .join(" ");
+    let rules = vec![port_rules.to_string(), ssh_rules.to_string()];
+    (name, rules)
+}
diff --git a/crates/vm-network-utils/src/main.rs b/crates/vm-network-utils/src/main.rs
new file mode 100644
index 0000000000..607802b116
--- /dev/null
+++ b/crates/vm-network-utils/src/main.rs
@@ -0,0 +1,16 @@
+use vm_network_utils::{clear_network, setup_network, NetworkSettings};
+
+fn main() {
+    let settings = NetworkSettings {
+        public_ip: "0.0.0.0".to_string(),
+        vm_ip: "127.0.0.1".to_string(),
+        bridge_name: "br0".to_string(),
+        port_range: (1000, 65535),
+        host_ssh_port: 2222,
+        vm_ssh_port: 22,
+    };
+    let test_name = "test-name";
+    println!("Clear: {:?}", clear_network(&settings, test_name));
+    //let result = setup_network(settings, test_name);
+    //println!("Setup: {result:?}");
+}

From 551026a2ca3e68b4828a6fe6272d657d6b340f06 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Wed, 21 Aug 2024 14:14:05 +0200
Subject: [PATCH 02/32] setup network on VM creation

---
 Cargo.lock                                  |   2 +
 Cargo.toml                                  |   1 +
 crates/server-config/src/lib.rs             |   4 +-
 crates/server-config/src/node_config.rs     |  44 +++-
 crates/server-config/src/resolved_config.rs |  22 +-
 crates/vm-network-utils/src/lib.rs          | 266 +++++++++-----------
 crates/vm-network-utils/src/main.rs         |   8 +-
 crates/workers/Cargo.toml                   |   1 +
 crates/workers/src/error.rs                 |   5 +
 crates/workers/src/workers.rs               | 122 +++++----
 nox/Cargo.toml                              |   1 +
 nox/src/node.rs                             |  28 ++-
 12 files changed, 286 insertions(+), 218 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9009f9e024..936b9d7d31 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5868,6 +5868,7 @@ dependencies = [
  "tracing-opentelemetry",
  "tracing-panic",
  "tracing-subscriber",
+ "vm-network-utils",
  "workers",
 ]
 
@@ -10277,6 +10278,7 @@ dependencies = [
  "toml_edit 0.22.18",
  "tracing",
  "types",
+ "vm-network-utils",
  "vm-utils",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index 2c288ec526..b6ece4bed3 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -104,6 +104,7 @@ types = { path = "crates/types" }
 core-distributor = { path = "crates/core-distributor" }
 log-format = { path = "crates/log-format" }
 vm-utils = { path = "crates/vm-utils" }
+vm-network-utils = { path = "crates/vm-network-utils" }
 
 # spell
 fluence-spell-dtos = "=0.7.5"
diff --git a/crates/server-config/src/lib.rs b/crates/server-config/src/lib.rs
index 9611f3b249..c8979b83e1 100644
--- a/crates/server-config/src/lib.rs
+++ b/crates/server-config/src/lib.rs
@@ -53,7 +53,9 @@ pub use resolved_config::ConfigData;
 pub use bootstrap_config::BootstrapConfig;
 pub use kademlia_config::KademliaConfig;
 pub use network_config::NetworkConfig;
-pub use node_config::{ChainConfig, ChainListenerConfig, Network, NodeConfig, TransportConfig};
+pub use node_config::{
+    ChainConfig, ChainListenerConfig, Network, NodeConfig, TransportConfig, VmNetworkConfig,
+};
 pub use resolved_config::TracingConfig;
 pub use resolved_config::{ResolvedConfig, UnresolvedConfig};
 pub use system_services_config::{AquaIpfsConfig, DeciderConfig, SystemServicesConfig};
diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index a6cb231a85..80dcb962bd 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -18,7 +18,7 @@
  */
 
 use std::collections::{BTreeMap, HashMap};
-use std::net::IpAddr;
+use std::net::{IpAddr, Ipv4Addr};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
@@ -614,15 +614,49 @@ pub struct DevModeConfig {
     pub binaries: BTreeMap<String, PathBuf>,
 }
 
+fn default_dev_mode_config() -> DevModeConfig {
+    DevModeConfig {
+        enable: false,
+        binaries: default_binaries_mapping(),
+    }
+}
+
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct VmConfig {
     pub libvirt_uri: String,
+    pub network: VmNetworkConfig,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
+pub struct VmNetworkConfig {
     pub bridge_name: String,
+    pub public_ip: Ipv4Addr,
+    pub vm_ip: Ipv4Addr,
+    #[serde(default = "default_host_ssh_port")]
+    pub host_ssh_port: u16,
+    #[serde(default = "default_vm_ssh_port")]
+    pub vm_ssh_port: u16,
+    #[serde(default = "default_port_range_config")]
+    pub port_range: PortRangeConfig,
 }
 
-fn default_dev_mode_config() -> DevModeConfig {
-    DevModeConfig {
-        enable: false,
-        binaries: default_binaries_mapping(),
+fn default_host_ssh_port() -> u16 {
+    2222
+}
+
+fn default_vm_ssh_port() -> u16 {
+    22
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
+pub struct PortRangeConfig {
+    pub start: u16,
+    pub end: u16,
+}
+
+fn default_port_range_config() -> PortRangeConfig {
+    PortRangeConfig {
+        start: 1000,
+        end: 65535,
     }
 }
diff --git a/crates/server-config/src/resolved_config.rs b/crates/server-config/src/resolved_config.rs
index d7e7fa2e54..79802bf617 100644
--- a/crates/server-config/src/resolved_config.rs
+++ b/crates/server-config/src/resolved_config.rs
@@ -250,6 +250,7 @@ fn process_args(raw_args: Vec<OsString>, data: Option<ConfigData>) -> eyre::Resu
 #[cfg(test)]
 mod tests {
     use std::io::Write;
+    use std::net::Ipv4Addr;
     use std::time::Duration;
 
     use base64::{engine::general_purpose::STANDARD as base64, Engine};
@@ -257,7 +258,7 @@ mod tests {
     use tempfile::{tempdir, NamedTempFile};
 
     use super::*;
-    use crate::node_config::VmConfig;
+    use crate::node_config::{PortRangeConfig, VmConfig, VmNetworkConfig};
     use crate::Network;
 
     #[test]
@@ -858,7 +859,14 @@ mod tests {
             r#"
             [vm]
             libvirt_uri = "qemu:///system"
+            [vm.network]
             bridge_name = "br422442"
+            public_ip = "1.1.1.1"
+            vm_ip = "2.2.2.2"
+            host_ssh_port = 2222
+            vm_ssh_port = 22
+            port_range.start = 1000
+            port_range.end = 65535
             "#
         )
         .expect("Could not write in file");
@@ -872,7 +880,17 @@ mod tests {
                 config.node_config.vm,
                 Some(VmConfig {
                     libvirt_uri: "qemu:///system".to_string(),
-                    bridge_name: "br422442".to_string()
+                    network: VmNetworkConfig {
+                        bridge_name: "br422442".to_string(),
+                        public_ip: Ipv4Addr::new(1, 1, 1, 1),
+                        vm_ip: Ipv4Addr::new(2, 2, 2, 2),
+                        port_range: PortRangeConfig {
+                            start: 1000,
+                            end: 65535,
+                        },
+                        host_ssh_port: 2222,
+                        vm_ssh_port: 22,
+                    }
                 })
             );
         });
diff --git a/crates/vm-network-utils/src/lib.rs b/crates/vm-network-utils/src/lib.rs
index 880fd841cf..16f53de933 100644
--- a/crates/vm-network-utils/src/lib.rs
+++ b/crates/vm-network-utils/src/lib.rs
@@ -1,47 +1,50 @@
 use iptables::IPTables;
+use std::net::Ipv4Addr;
 use thiserror::Error;
 
 pub struct NetworkSettings {
-    pub public_ip: String,
-    pub vm_ip: String,
+    pub public_ip: Ipv4Addr,
+    pub vm_ip: Ipv4Addr,
     pub bridge_name: String,
     pub port_range: (u16, u16),
     pub host_ssh_port: u16,
     pub vm_ssh_port: u16,
 }
 
-#[derive(Debug, Error)]
-#[error(transparent)]
-pub struct DNatSetupError(#[from] Box<dyn std::error::Error>);
-
-#[derive(Debug, Error)]
-#[error(transparent)]
-pub struct SNatSetupError(#[from] Box<dyn std::error::Error>);
-
-#[derive(Debug, Error)]
-#[error(transparent)]
-pub struct FwdSetupError(#[from] Box<dyn std::error::Error>);
-
+// The iptables crate only return this kind of error, which can't be sent between threads
+// So, the easiest solution is to store the error message as strings
 #[derive(Debug, Error)]
 pub enum NetworkSetupError {
-    #[error("error setting up DNAT rules: {0}")]
-    DNat(#[from] DNatSetupError),
-    #[error("error setting up SNAT rules: {0}")]
-    SNat(#[from] SNatSetupError),
-    #[error("error setting up FWD rules: {0}")]
-    Fwd(#[from] FwdSetupError),
-    #[error("error initializing iptables: {0}")]
-    Init(#[from] Box<dyn std::error::Error>),
+    #[error("error setting up DNAT rules: {message}")]
+    DNat { message: String },
+    #[error("error setting up SNAT rules: {message}")]
+    SNat { message: String },
+    #[error("error setting up FWD rules: {message}")]
+    Fwd { message: String },
+    #[error("error initializing iptables: {message}")]
+    Init { message: String },
+    #[error("error cleaning the rules for the chain {chain_name}: {message}")]
+    Clean { chain_name: String, message: String },
 }
 
+type IpTablesError = Box<dyn std::error::Error>;
+
 pub fn setup_network(
-    network_settings: NetworkSettings,
+    network_settings: &NetworkSettings,
     name: &str,
 ) -> Result<(), NetworkSetupError> {
-    let ipt = iptables::new(false)?;
-    setup_snat(&network_settings, &ipt, name)?;
-    setup_dnat(&network_settings, &ipt, name)?;
-    setup_fwd(&network_settings, &ipt, name)?;
+    let ipt = iptables::new(false).map_err(|err| NetworkSetupError::Init {
+        message: err.to_string(),
+    })?;
+    setup_snat(network_settings, &ipt, name).map_err(|err| NetworkSetupError::SNat {
+        message: err.to_string(),
+    })?;
+    setup_dnat(network_settings, &ipt, name).map_err(|err| NetworkSetupError::DNat {
+        message: err.to_string(),
+    })?;
+    setup_fwd(network_settings, &ipt, name).map_err(|err| NetworkSetupError::Fwd {
+        message: err.to_string(),
+    })?;
     Ok(())
 }
 
@@ -49,35 +52,35 @@ pub fn clear_network(
     network_settings: &NetworkSettings,
     name: &str,
 ) -> Result<(), NetworkSetupError> {
+    clear_network_inner(network_settings, name).map_err(|err| NetworkSetupError::Clean {
+        chain_name: name.to_string(),
+        message: err.to_string(),
+    })
+}
+
+fn clear_network_inner(
+    network_settings: &NetworkSettings,
+    name: &str,
+) -> Result<(), IpTablesError> {
     let ipt = iptables::new(false)?;
-    let snat_result = {
+    {
         let (chain_name, rules) = snat_rules(network_settings, name);
-        clear_chain(&ipt, &chain_name, &rules)
-    };
+        clear_chain(&ipt, &chain_name, &rules)?;
+    }
 
-    let dnat_result = {
+    {
         let (chain_name, rules) = dnat_rules(network_settings, name);
-        clear_chain(&ipt, &chain_name, &rules)
-    };
+        clear_chain(&ipt, &chain_name, &rules)?;
+    }
 
-    let fwd_result = {
+    {
         let (chain_name, rules) = fwd_rules(network_settings, name);
-        clear_chain(&ipt, &chain_name, &rules)
-    };
-
-    // Try to clean as much as possible and only then fail
-    snat_result?;
-    dnat_result?;
-    fwd_result?;
-
+        clear_chain(&ipt, &chain_name, &rules)?;
+    }
     Ok(())
 }
 
-fn clear_chain(
-    ipt: &IPTables,
-    chain_name: &str,
-    rules: &Vec<String>,
-) -> Result<(), NetworkSetupError> {
+fn clear_chain(ipt: &IPTables, chain_name: &str, rules: &Vec<String>) -> Result<(), IpTablesError> {
     let exists = ipt.chain_exists("nat", chain_name)?;
     if !exists {
         tracing::warn!("Can't clean the chain {chain_name}: doesn't exist");
@@ -102,7 +105,7 @@ fn setup_dnat(
     network_settings: &NetworkSettings,
     ipt: &IPTables,
     name: &str,
-) -> Result<(), DNatSetupError> {
+) -> Result<(), IpTablesError> {
     let (chain_name, rules) = dnat_rules(network_settings, name);
     apply_rules(ipt, &chain_name, &rules)?;
     Ok(())
@@ -112,7 +115,7 @@ fn setup_snat(
     network_settings: &NetworkSettings,
     ipt: &IPTables,
     name: &str,
-) -> Result<(), SNatSetupError> {
+) -> Result<(), IpTablesError> {
     let (chain_name, rules) = snat_rules(network_settings, name);
     apply_rules(ipt, &chain_name, &rules)?;
     Ok(())
@@ -122,17 +125,13 @@ fn setup_fwd(
     network_settings: &NetworkSettings,
     ipt: &IPTables,
     name: &str,
-) -> Result<(), FwdSetupError> {
+) -> Result<(), IpTablesError> {
     let (chain_name, rules) = fwd_rules(network_settings, name);
     apply_rules(ipt, &chain_name, &rules)?;
     Ok(())
 }
 
-fn apply_rules(
-    ipt: &IPTables,
-    name: &str,
-    rules: &Vec<String>,
-) -> Result<(), Box<dyn std::error::Error>> {
+fn apply_rules(ipt: &IPTables, name: &str, rules: &Vec<String>) -> Result<(), IpTablesError> {
     ipt.new_chain("nat", &name)?;
     for rule in rules {
         ipt.append("nat", &name, &rule)?;
@@ -170,56 +169,34 @@ fn apply_rules(
 //          -j MASQUERADE
 // ```
 fn snat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+    let public_ip = network_settings.public_ip;
+    let vm_ip = network_settings.vm_ip;
+    let vm_ssh_port = network_settings.vm_ssh_port;
+    let host_ssh_port = network_settings.host_ssh_port;
+    let port_start = network_settings.port_range.0;
+    let port_end = network_settings.port_range.1;
+
     let name = format!("SNAT-{name}");
-    let common_prefix = &[
-        format!("-s {}", network_settings.vm_ip),
-        "-p tcp".to_string(),
-        "-m tcp".to_string(),
-    ]
-    .join(" ");
-
-    let ports_rules = &[
-        common_prefix.clone(),
-        format!(
-            "--dport {}:{}",
-            network_settings.port_range.0, network_settings.port_range.1
-        ),
-        "-j SNAT".to_string(),
-        format!("--to-source {}", network_settings.public_ip),
-    ]
-    .join(" ");
-
-    let ssh_ports_rules = &[
-        common_prefix.clone(),
-        format!("--dport {}", network_settings.vm_ssh_port),
-        "-j SNAT".to_string(),
-        format!("--to-source {}", network_settings.public_ip),
-    ]
-    .join(" ");
-
-    let masquerade_ports_rules = &[
-        common_prefix.clone(),
-        format!(
-            "--dport {}:{}",
-            network_settings.port_range.0, network_settings.port_range.1
-        ),
-        format!("-d {}", network_settings.public_ip),
-        "-j MASQUERADE".to_string(),
-    ]
-    .join(" ");
-    let masquerade_ssh_ports_rules = &[
-        common_prefix.clone(),
-        format!("--dport {}", network_settings.host_ssh_port),
-        format!("-d {}", network_settings.public_ip),
-        "-j MASQUERADE".to_string(),
-    ]
-    .join(" ");
+
+    let ports_rules = format!(
+        "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j SNAT --to-source {public_ip}"
+    );
+
+    let ssh_ports_rules =
+        format!("-s {vm_ip} -p tcp -m tcp --dport {vm_ssh_port} -j SNAT --to-source {public_ip}");
+
+    let masquerade_ports_rules = format!(
+        "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -d {public_ip} -j MASQUERADE"
+    );
+
+    let masquerade_ssh_ports_rules =
+        format!("-s {vm_ip} -p tcp -m tcp --dport {host_ssh_port} -d {public_ip} -j MASQUERADE");
 
     let rules = vec![
-        ports_rules.to_string(),
-        ssh_ports_rules.to_string(),
-        masquerade_ports_rules.to_string(),
-        masquerade_ssh_ports_rules.to_string(),
+        ports_rules,
+        ssh_ports_rules,
+        masquerade_ports_rules,
+        masquerade_ssh_ports_rules,
     ];
 
     (name, rules)
@@ -245,36 +222,25 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
 //           -j DNAT
 // ```
 fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+    let public_ip = network_settings.public_ip;
+    let vm_ip = network_settings.vm_ip;
+    let vm_ssh_port = network_settings.vm_ssh_port;
+    let host_ssh_port = network_settings.host_ssh_port;
+    let port_start = network_settings.port_range.0;
+    let port_end = network_settings.port_range.1;
+
     let name = format!("DNAT-{name}");
-    let common_prefix = &[
-        format!("-d {}", network_settings.public_ip),
-        "-p tcp".to_string(),
-        "-m tcp".to_string(),
-        format!(
-            "--dport {}:{}",
-            network_settings.port_range.0, network_settings.port_range.1
-        ),
-        "-j DNAT".to_string(),
-    ]
-    .join(" ");
-
-    let port_rules = &[
-        common_prefix.clone(),
-        format!(
-            "--to-destination {}:{}-{}",
-            network_settings.vm_ip, network_settings.port_range.0, network_settings.port_range.1
-        ),
-    ]
-    .join(" ");
-    let ssh_port_rules = &[
-        common_prefix.clone(),
-        format!(
-            "--to-destination {}:{}",
-            network_settings.vm_ip, network_settings.vm_ssh_port
-        ),
-    ]
-    .join(" ");
-    let rules = vec![port_rules.to_string(), ssh_port_rules.to_string()];
+
+    let port_rules = format!(
+        "-d {public_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j DNAT \
+        --to-destination {vm_ip}:{port_start}-{port_end}",
+    );
+
+    let ssh_port_rules = format!(
+        "-d {public_ip} -p tcp -m tcp --dport {host_ssh_port} -j DNAT \
+        --to-destination {vm_ip}:{vm_ssh_port}",
+    );
+    let rules = vec![port_rules, ssh_port_rules];
     (name, rules)
 }
 
@@ -294,29 +260,21 @@ fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
 //          -j ACCEPT
 // ```
 fn fwd_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+    let vm_ip = network_settings.vm_ip;
+    let vm_ssh_port = network_settings.vm_ssh_port;
+    let port_start = network_settings.port_range.0;
+    let port_end = network_settings.port_range.1;
+    let bridge_name = &network_settings.bridge_name;
+
     let name = format!("FWD-{name}");
-    let common_prefix = &[
-        format!("-d {}", network_settings.public_ip),
-        "-p tcp".to_string(),
-        "-m tcp".to_string(),
-        format!("-o {}", network_settings.bridge_name),
-    ]
-    .join(" ");
-    let port_rules = &[
-        common_prefix.clone(),
-        format!(
-            "--dport {}:{}",
-            network_settings.port_range.0, network_settings.port_range.1
-        ),
-        "-j ACCEPT".to_string(),
-    ]
-    .join(" ");
-    let ssh_rules = &[
-        common_prefix.clone(),
-        format!("--dport {}", network_settings.vm_ssh_port),
-        "-j ACCEPT".to_string(),
-    ]
-    .join(" ");
-    let rules = vec![port_rules.to_string(), ssh_rules.to_string()];
+
+    let port_rules = format!(
+        "-d {vm_ip} -o {bridge_name} -p tcp -m tcp --dport {port_start}:{port_end} -j ACCEPT"
+    );
+
+    let ssh_port_rules =
+        format!("-d {vm_ip} -o {bridge_name} -p tcp -m tcp --dport {vm_ssh_port} -j ACCEPT");
+
+    let rules = vec![port_rules, ssh_port_rules];
     (name, rules)
 }
diff --git a/crates/vm-network-utils/src/main.rs b/crates/vm-network-utils/src/main.rs
index 607802b116..d13b6dc607 100644
--- a/crates/vm-network-utils/src/main.rs
+++ b/crates/vm-network-utils/src/main.rs
@@ -1,9 +1,11 @@
+use std::net::Ipv4Addr;
+use std::str::FromStr;
 use vm_network_utils::{clear_network, setup_network, NetworkSettings};
 
 fn main() {
     let settings = NetworkSettings {
-        public_ip: "0.0.0.0".to_string(),
-        vm_ip: "127.0.0.1".to_string(),
+        public_ip: Ipv4Addr::from_str("0.0.0.0").unwrap(),
+        vm_ip: Ipv4Addr::from_str("127.0.0.1").unwrap(),
         bridge_name: "br0".to_string(),
         port_range: (1000, 65535),
         host_ssh_port: 2222,
@@ -11,6 +13,6 @@ fn main() {
     };
     let test_name = "test-name";
     println!("Clear: {:?}", clear_network(&settings, test_name));
-    //let result = setup_network(settings, test_name);
+    // let result = setup_network(settings, test_name);
     //println!("Setup: {result:?}");
 }
diff --git a/crates/workers/Cargo.toml b/crates/workers/Cargo.toml
index 0f095c114e..75c015146d 100644
--- a/crates/workers/Cargo.toml
+++ b/crates/workers/Cargo.toml
@@ -24,6 +24,7 @@ tokio = { workspace = true, features = ["fs", "sync"] }
 derivative = { workspace = true }
 types = { workspace = true }
 vm-utils = { workspace = true }
+vm-network-utils = { workspace = true }
 async-trait = "0.1.79"
 tokio-stream = { workspace = true, features = ["fs"] }
 
diff --git a/crates/workers/src/error.rs b/crates/workers/src/error.rs
index 24beefbc43..61ddc6176e 100644
--- a/crates/workers/src/error.rs
+++ b/crates/workers/src/error.rs
@@ -23,6 +23,7 @@ use std::path::PathBuf;
 use thiserror::Error;
 use types::peer_scope::WorkerId;
 use types::DealId;
+use vm_network_utils::NetworkSetupError;
 use vm_utils::VmError;
 
 #[derive(Debug, Error)]
@@ -177,4 +178,8 @@ pub enum WorkersError {
     VmNotFound(WorkerId),
     #[error(transparent)]
     VmError(#[from] VmError),
+    #[error("Only one VM allowed on the host, and a VM already exists for worker {0}")]
+    VMAlreadyExists(WorkerId),
+    #[error("Error while setting up VM network: {0}")]
+    VmNetworkError(#[from] NetworkSetupError),
 }
diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index 47ed2ef581..98eaddf079 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -36,6 +36,7 @@ use core_distributor::{CoreDistributor, ThreadPinner, CUID};
 use fluence_libp2p::PeerId;
 use types::peer_scope::WorkerId;
 use types::DealId;
+use vm_network_utils::NetworkSettings;
 use vm_utils::{CreateVMDomainParams, NonEmpty, VmStatus};
 
 const WORKER_DATA_DIR: &str = "data";
@@ -122,14 +123,14 @@ impl WorkersConfig {
 pub struct VmConfig {
     /// Uri to the libvirt API
     libvirt_uri: String,
-    bridge_name: String,
+    network: NetworkSettings,
 }
 
 impl VmConfig {
-    pub fn new(libvirt_uri: String, bridge_name: String) -> Self {
+    pub fn new(libvirt_uri: String, network: NetworkSettings) -> Self {
         Self {
             libvirt_uri,
-            bridge_name,
+            network,
         }
     }
 }
@@ -226,6 +227,10 @@ impl Workers {
             .map(|info| info.deal_id.clone())
     }
 
+    pub fn vm_worker_exists(&self) -> bool {
+        todo!()
+    }
+
     /// Creates a new worker with the given `deal_id` and initial peer ID.
     ///
     /// # Arguments
@@ -647,58 +652,85 @@ impl Workers {
         match &self.config.vm {
             None => Err(WorkersError::FeatureDisabled),
             Some(vm_config) => {
-                let assignment = {
-                    let guard = self.assignments.read();
-                    let assignment = guard
-                        .get(&worker_id)
-                        .ok_or_else(|| WorkersError::WorkerNotFound(worker_id))?;
-
-                    NonEmpty::from_vec(assignment.logical_core_ids())
-                        .ok_or_else(|| WorkersError::WrongAssignment)?
-                };
-
-                let file_name = &image
-                    .file_name()
-                    .ok_or_else(|| WorkersError::VMImageNotFile {
-                        image: image.to_path_buf(),
-                    })?;
-
-                let vm_name = worker_id.to_string();
-
-                let worker_image = self
-                    .workers_dir
-                    .join(&vm_name)
-                    .join(WORKER_DATA_DIR)
-                    .join(file_name);
-
-                tokio::fs::copy(&image, &worker_image)
-                    .await
-                    .map_err(|err| WorkersError::FailedToCopyVMImage {
-                        image: image.to_path_buf(),
-                        err,
-                    })?;
-
-                let params = CreateVMDomainParams::new(
-                    vm_name.clone(),
-                    worker_image,
-                    assignment,
-                    vm_config.bridge_name.clone(),
-                );
+                let vm_created = self
+                    .worker_infos
+                    .read()
+                    .values()
+                    .any(|worker_info| *worker_info.vm_flag.read());
+                if vm_created {
+                    return Err(WorkersError::VMAlreadyExists(worker_id));
+                }
 
-                vm_utils::create_domain(vm_config.libvirt_uri.clone().as_str(), &params)?;
+                let vm_name = self.create_vm_inner(worker_id, image, vm_config).await?;
+                Ok(vm_name)
+            }
+        }
+    }
 
-                self.set_vm_flag(worker_id, true).await?;
+    async fn create_vm_inner(
+        &self,
+        worker_id: WorkerId,
+        image: &Path,
+        vm_config: &VmConfig,
+    ) -> Result<String, WorkersError> {
+        let assignment = {
+            let guard = self.assignments.read();
+            let assignment = guard
+                .get(&worker_id)
+                .ok_or_else(|| WorkersError::WorkerNotFound(worker_id))?;
 
-                vm_utils::start_vm(vm_config.libvirt_uri.as_str(), vm_name.as_str())?;
+            NonEmpty::from_vec(assignment.logical_core_ids())
+                .ok_or_else(|| WorkersError::WrongAssignment)?
+        };
 
-                Ok(vm_name)
-            }
+        let file_name = &image
+            .file_name()
+            .ok_or_else(|| WorkersError::VMImageNotFile {
+                image: image.to_path_buf(),
+            })?;
+
+        let vm_name = worker_id.to_string();
+
+        let worker_image = self
+            .workers_dir
+            .join(&vm_name)
+            .join(WORKER_DATA_DIR)
+            .join(file_name);
+
+        tokio::fs::copy(&image, &worker_image)
+            .await
+            .map_err(|err| WorkersError::FailedToCopyVMImage {
+                image: image.to_path_buf(),
+                err,
+            })?;
+
+        let params = CreateVMDomainParams::new(
+            vm_name.clone(),
+            worker_image,
+            assignment,
+            vm_config.network.bridge_name.clone(),
+        );
+
+        vm_utils::create_domain(vm_config.libvirt_uri.clone().as_str(), &params)?;
+
+        self.set_vm_flag(worker_id, true).await?;
+        // First, create the network
+        vm_network_utils::setup_network(&vm_config.network, vm_name.as_str())?;
+        // And only then start the VM
+        let result = vm_utils::start_vm(vm_config.libvirt_uri.as_str(), vm_name.as_str());
+        if let Err(err) = result {
+            // Clear the network on errors so we can retry later (setup_network isn't idempotent)
+            vm_network_utils::clear_network(&vm_config.network, vm_name.as_str())?;
+            return Err(WorkersError::VmError(err));
         }
+
+        Ok(vm_name)
     }
 
     fn remove_vm(&self, worker_id: WorkerId) -> Result<(), WorkersError> {
         if let Some(vm_config) = &self.config.vm {
             if self.has_vm(worker_id)? {
+                vm_network_utils::clear_network(&vm_config.network, &worker_id.to_string())?;
                 vm_utils::remove_domain(
                     vm_config.libvirt_uri.as_str(),
                     worker_id.to_string().as_str(),
diff --git a/nox/Cargo.toml b/nox/Cargo.toml
index 83805138ff..5f2298928b 100644
--- a/nox/Cargo.toml
+++ b/nox/Cargo.toml
@@ -73,6 +73,7 @@ log-format = { workspace = true }
 thiserror = { workspace = true }
 cpu-utils = { workspace = true }
 cfg-if = { workspace = true }
+vm-network-utils = { workspace = true }
 
 [dev-dependencies]
 parking_lot = { workspace = true }
diff --git a/nox/src/node.rs b/nox/src/node.rs
index 820fbb7083..84ae4d3b92 100644
--- a/nox/src/node.rs
+++ b/nox/src/node.rs
@@ -198,14 +198,13 @@ impl<RT: AquaRuntime> Node<RT> {
             key_storage.clone(),
         );
 
-        let workers_config = WorkersConfig::new(
-            config.node_config.workers_queue_buffer,
-            config
-                .node_config
-                .vm
-                .clone()
-                .map(|conf| VmConfig::new(conf.libvirt_uri, conf.bridge_name)),
-        );
+        let workers_config =
+            WorkersConfig::new(
+                config.node_config.workers_queue_buffer,
+                config.node_config.vm.clone().map(|conf| {
+                    VmConfig::new(conf.libvirt_uri, to_vm_network_settings(conf.network))
+                }),
+            );
 
         let (workers, worker_events) = Workers::from_path(
             workers_config,
@@ -568,6 +567,19 @@ impl<RT: AquaRuntime> Node<RT> {
     }
 }
 
+fn to_vm_network_settings(
+    config: server_config::VmNetworkConfig,
+) -> vm_network_utils::NetworkSettings {
+    vm_network_utils::NetworkSettings {
+        public_ip: config.public_ip,
+        vm_ip: config.vm_ip,
+        bridge_name: config.bridge_name,
+        port_range: (config.port_range.start, config.port_range.end),
+        host_ssh_port: config.host_ssh_port,
+        vm_ssh_port: config.vm_ssh_port,
+    }
+}
+
 pub struct StartedNode {
     pub cancellation_token: CancellationToken,
     pub exit_outlet: oneshot::Sender<()>,

From b63d4ef4c75b244661a4021064d0b098f80e5fc4 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Wed, 21 Aug 2024 15:19:20 +0200
Subject: [PATCH 03/32] remove trash

---
 crates/vm-network-utils/src/main.rs | 18 ------------------
 crates/workers/src/workers.rs       |  4 ----
 2 files changed, 22 deletions(-)
 delete mode 100644 crates/vm-network-utils/src/main.rs

diff --git a/crates/vm-network-utils/src/main.rs b/crates/vm-network-utils/src/main.rs
deleted file mode 100644
index d13b6dc607..0000000000
--- a/crates/vm-network-utils/src/main.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-use std::net::Ipv4Addr;
-use std::str::FromStr;
-use vm_network_utils::{clear_network, setup_network, NetworkSettings};
-
-fn main() {
-    let settings = NetworkSettings {
-        public_ip: Ipv4Addr::from_str("0.0.0.0").unwrap(),
-        vm_ip: Ipv4Addr::from_str("127.0.0.1").unwrap(),
-        bridge_name: "br0".to_string(),
-        port_range: (1000, 65535),
-        host_ssh_port: 2222,
-        vm_ssh_port: 22,
-    };
-    let test_name = "test-name";
-    println!("Clear: {:?}", clear_network(&settings, test_name));
-    // let result = setup_network(settings, test_name);
-    //println!("Setup: {result:?}");
-}
diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index 98eaddf079..f23b27503e 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -227,10 +227,6 @@ impl Workers {
             .map(|info| info.deal_id.clone())
     }
 
-    pub fn vm_worker_exists(&self) -> bool {
-        todo!()
-    }
-
     /// Creates a new worker with the given `deal_id` and initial peer ID.
     ///
     /// # Arguments

From 5f165ab39c9067aa45c007c6bd35cf7dfc521d0e Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Wed, 21 Aug 2024 15:43:36 +0200
Subject: [PATCH 04/32] fix clippy

---
 crates/vm-network-utils/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/crates/vm-network-utils/src/lib.rs b/crates/vm-network-utils/src/lib.rs
index 16f53de933..f3cd25aa5d 100644
--- a/crates/vm-network-utils/src/lib.rs
+++ b/crates/vm-network-utils/src/lib.rs
@@ -132,9 +132,9 @@ fn setup_fwd(
 }
 
 fn apply_rules(ipt: &IPTables, name: &str, rules: &Vec<String>) -> Result<(), IpTablesError> {
-    ipt.new_chain("nat", &name)?;
+    ipt.new_chain("nat", name)?;
     for rule in rules {
-        ipt.append("nat", &name, &rule)?;
+        ipt.append("nat", name, rule)?;
     }
     Ok(())
 }

From 13f5ad1cfaa2fd29a26d4517e6881b77484a5186 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Wed, 21 Aug 2024 16:10:14 +0200
Subject: [PATCH 05/32] fix 28 chain name char restriction

---
 crates/vm-network-utils/src/lib.rs | 12 +++++++++---
 crates/workers/src/workers.rs      | 18 +++++++++++-------
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/crates/vm-network-utils/src/lib.rs b/crates/vm-network-utils/src/lib.rs
index f3cd25aa5d..70e05ac6cb 100644
--- a/crates/vm-network-utils/src/lib.rs
+++ b/crates/vm-network-utils/src/lib.rs
@@ -176,7 +176,7 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
     let port_start = network_settings.port_range.0;
     let port_end = network_settings.port_range.1;
 
-    let name = format!("SNAT-{name}");
+    let name = cut_chain_name(format!("SNAT-{name}"));
 
     let ports_rules = format!(
         "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j SNAT --to-source {public_ip}"
@@ -229,7 +229,7 @@ fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
     let port_start = network_settings.port_range.0;
     let port_end = network_settings.port_range.1;
 
-    let name = format!("DNAT-{name}");
+    let name = cut_chain_name(format!("DNAT-{name}"));
 
     let port_rules = format!(
         "-d {public_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j DNAT \
@@ -266,7 +266,7 @@ fn fwd_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<Str
     let port_end = network_settings.port_range.1;
     let bridge_name = &network_settings.bridge_name;
 
-    let name = format!("FWD-{name}");
+    let name = cut_chain_name(format!("FWD-{name}"));
 
     let port_rules = format!(
         "-d {vm_ip} -o {bridge_name} -p tcp -m tcp --dport {port_start}:{port_end} -j ACCEPT"
@@ -278,3 +278,9 @@ fn fwd_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<Str
     let rules = vec![port_rules, ssh_port_rules];
     (name, rules)
 }
+
+// iptables allows only 29 characters for the chain name
+fn cut_chain_name(mut name: String) -> String {
+    name.truncate(28);
+    name
+}
diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index f23b27503e..0bb0eaf27f 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -709,17 +709,21 @@ impl Workers {
 
         vm_utils::create_domain(vm_config.libvirt_uri.clone().as_str(), &params)?;
 
-        self.set_vm_flag(worker_id, true).await?;
-        // First, create the network
-        vm_network_utils::setup_network(&vm_config.network, vm_name.as_str())?;
-        // And only then start the VM
-        let result = vm_utils::start_vm(vm_config.libvirt_uri.as_str(), vm_name.as_str());
+        let result: Result<_, WorkersError> = try {
+            // First, create the network
+            vm_network_utils::setup_network(&vm_config.network, vm_name.as_str())?;
+            // And only then start the VM
+            vm_utils::start_vm(vm_config.libvirt_uri.as_str(), vm_name.as_str())?;
+        };
         if let Err(err) = result {
-            // Clear the network on errors so we can retry later (setup_network isn't idempotent)
+            // Clear the network on errors, so we can retry later (setup_network isn't idempotent)
+            tracing::warn!("couldn't create network or start VM, cleaning up network");
             vm_network_utils::clear_network(&vm_config.network, vm_name.as_str())?;
-            return Err(WorkersError::VmError(err));
+            return Err(err);
         }
 
+        self.set_vm_flag(worker_id, true).await?;
+
         Ok(vm_name)
     }
 

From 0217463a2df8be6785416fc93bdf4c16b2596b5b Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 22 Aug 2024 16:57:48 +0200
Subject: [PATCH 06/32] remove ref

---
 .github/workflows/e2e.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 71d1491ea4..918a9ae679 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -84,7 +84,6 @@ jobs:
       - nox-snapshot
     uses: fluencelabs/cli/.github/workflows/tests.yml@main
     with:
-      ref: up-clients
       nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}"
 
   js-client:

From bf03bc4ec8b2756263cfbba61c6ebe500890eb2f Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 22 Aug 2024 16:35:13 +0200
Subject: [PATCH 07/32] feat(vm): add all GPUs on the host to a VM

---
 Cargo.lock                                   | 197 ++++++++++++++++---
 Cargo.toml                                   |   3 +-
 crates/gpu-utils/Cargo.toml                  |  10 +
 crates/gpu-utils/src/lib.rs                  |  38 ++++
 crates/gpu-utils/x1                          |  45 +++++
 crates/gpu-utils/x2                          |  45 +++++
 crates/vm-utils/Cargo.toml                   |   2 +
 crates/vm-utils/src/template.xml             |   1 +
 crates/vm-utils/src/vm_utils.rs              |  56 +++++-
 crates/vm-utils/tests/expected_vm_config.xml |  12 ++
 10 files changed, 375 insertions(+), 34 deletions(-)
 create mode 100644 crates/gpu-utils/Cargo.toml
 create mode 100644 crates/gpu-utils/src/lib.rs
 create mode 100644 crates/gpu-utils/x1
 create mode 100644 crates/gpu-utils/x2

diff --git a/Cargo.lock b/Cargo.lock
index 936b9d7d31..0dc9bec821 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1528,7 +1528,7 @@ dependencies = [
  "num-traits",
  "serde",
  "wasm-bindgen",
- "windows-targets 0.52.0",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -1810,9 +1810,9 @@ dependencies = [
 
 [[package]]
 name = "core-foundation"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -3131,6 +3131,14 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "gpu-utils"
+version = "0.1.0"
+dependencies = [
+ "pci-info",
+ "thiserror",
+]
+
 [[package]]
 name = "group"
 version = "0.13.0"
@@ -3612,7 +3620,7 @@ dependencies = [
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "windows-core",
+ "windows-core 0.51.1",
 ]
 
 [[package]]
@@ -3682,7 +3690,7 @@ dependencies = [
  "rtnetlink",
  "system-configuration",
  "tokio",
- "windows",
+ "windows 0.51.1",
 ]
 
 [[package]]
@@ -6493,6 +6501,17 @@ dependencies = [
  "nom",
 ]
 
+[[package]]
+name = "pci-info"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3993ca73bbb74f755ab5c7ef3e07d317e51bc10293bf9d61065ab34e724e802e"
+dependencies = [
+ "core-foundation",
+ "windows 0.56.0",
+ "wmi",
+]
+
 [[package]]
 name = "peer-metrics"
 version = "0.1.0"
@@ -9228,6 +9247,7 @@ name = "vm-utils"
 version = "0.1.0"
 dependencies = [
  "ccp-shared",
+ "gpu-utils",
  "log-utils",
  "mac_address",
  "nonempty",
@@ -10029,10 +10049,32 @@ version = "0.51.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9"
 dependencies = [
- "windows-core",
+ "windows-core 0.51.1",
  "windows-targets 0.48.5",
 ]
 
+[[package]]
+name = "windows"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
+dependencies = [
+ "windows-core 0.52.0",
+ "windows-implement 0.52.0",
+ "windows-interface 0.52.0",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132"
+dependencies = [
+ "windows-core 0.56.0",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-core"
 version = "0.51.1"
@@ -10042,6 +10084,80 @@ dependencies = [
  "windows-targets 0.48.5",
 ]
 
+[[package]]
+name = "windows-core"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6"
+dependencies = [
+ "windows-implement 0.56.0",
+ "windows-interface 0.56.0",
+ "windows-result",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12168c33176773b86799be25e2a2ba07c7aab9968b37541f1094dbd7a60c8946"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.46",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.46",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d8dc32e0095a7eeccebd0e3f09e9509365ecb3fc6ac4d6f5f14a3f6392942d1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.46",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.46",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -10057,7 +10173,7 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.0",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -10077,17 +10193,18 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.0",
- "windows_aarch64_msvc 0.52.0",
- "windows_i686_gnu 0.52.0",
- "windows_i686_msvc 0.52.0",
- "windows_x86_64_gnu 0.52.0",
- "windows_x86_64_gnullvm 0.52.0",
- "windows_x86_64_msvc 0.52.0",
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
 ]
 
 [[package]]
@@ -10098,9 +10215,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -10110,9 +10227,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -10122,9 +10239,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -10134,9 +10257,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -10146,9 +10269,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
@@ -10158,9 +10281,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -10170,9 +10293,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
@@ -10252,6 +10375,20 @@ dependencies = [
  "wast 35.0.2",
 ]
 
+[[package]]
+name = "wmi"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc2f0a4062ca522aad4705a2948fd4061b3857537990202a8ddd5af21607f79a"
+dependencies = [
+ "chrono",
+ "futures",
+ "log",
+ "serde",
+ "thiserror",
+ "windows 0.52.0",
+]
+
 [[package]]
 name = "workers"
 version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index b6ece4bed3..d1c600b3be 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -49,7 +49,7 @@ members = [
     "crates/core-distributor",
     "crates/log-format",
     "crates/vm-utils",
-    "crates/vm-network-utils",
+    "crates/vm-network-utils", "crates/gpu-utils",
 ]
 exclude = [
     "nox/tests/tetraplets",
@@ -105,6 +105,7 @@ core-distributor = { path = "crates/core-distributor" }
 log-format = { path = "crates/log-format" }
 vm-utils = { path = "crates/vm-utils" }
 vm-network-utils = { path = "crates/vm-network-utils" }
+gpu-utils = { path = "crates/gpu-utils" }
 
 # spell
 fluence-spell-dtos = "=0.7.5"
diff --git a/crates/gpu-utils/Cargo.toml b/crates/gpu-utils/Cargo.toml
new file mode 100644
index 0000000000..03cba6d18f
--- /dev/null
+++ b/crates/gpu-utils/Cargo.toml
@@ -0,0 +1,10 @@
+[package]
+name = "gpu-utils"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+pci-info = "0.1.0"
+thiserror = { workspace = true }
\ No newline at end of file
diff --git a/crates/gpu-utils/src/lib.rs b/crates/gpu-utils/src/lib.rs
new file mode 100644
index 0000000000..50cf674191
--- /dev/null
+++ b/crates/gpu-utils/src/lib.rs
@@ -0,0 +1,38 @@
+use pci_info::pci_enums::PciDeviceClass::DisplayController;
+use pci_info::{PciDeviceEnumerationError, PciInfo, PciInfoError, PciInfoPropertyError};
+use thiserror::Error;
+
+pub use pci_info::PciLocation;
+
+#[derive(Debug, Error)]
+pub enum PciError {
+    #[error("can't get list of devices: {0}")]
+    PciInfoError(#[from] PciInfoError),
+    #[error("can't get info for a device: {0}")]
+    PciInfoEnumerationError(#[from] PciDeviceEnumerationError),
+    #[error("can't get properties for a device: {0}")]
+    PciInfoPropertyError(Box<PciInfoError>),
+    #[error("required property is not support for a device")]
+    UnsupportedProperty,
+}
+
+pub fn get_gpu_pci() -> Result<Vec<PciLocation>, PciError> {
+    let info = PciInfo::enumerate_pci()?;
+    let mut result = Vec::new();
+    for device in info {
+        let device = device?;
+        let device_class = process_property_result(device.device_class())?;
+        if device_class == DisplayController {
+            result.push(process_property_result(device.location())?);
+        }
+    }
+    Ok(result)
+}
+
+fn process_property_result<T>(result: Result<T, &PciInfoPropertyError>) -> Result<T, PciError> {
+    match result {
+        Ok(device) => Ok(device),
+        Err(PciInfoPropertyError::Unsupported) => Err(PciError::UnsupportedProperty),
+        Err(PciInfoPropertyError::Error(err)) => Err(PciError::PciInfoPropertyError(err.clone())),
+    }
+}
diff --git a/crates/gpu-utils/x1 b/crates/gpu-utils/x1
new file mode 100644
index 0000000000..562d708066
--- /dev/null
+++ b/crates/gpu-utils/x1
@@ -0,0 +1,45 @@
+<domain type='kvm'>
+    <name>test-id</name>
+    <memory unit='KiB'>8388608</memory>
+    <vcpu placement='static'>2</vcpu>
+    <cputune>
+        <vcpupin vcpu='0' cpuset='1'/>
+        <vcpupin vcpu='1' cpuset='8'/>
+    </cputune>
+    <os>
+        <type arch='x86_64' machine='pc-i440fx-2.9'>hvm</type>
+        <boot dev='hd'/>
+    </os>
+    <devices>
+        <disk type='file' device='disk'>
+            <driver name='qemu' type='qcow2'/>
+            <source file='test-image'/>
+            <target dev='vda' bus='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+        </disk>
+        <interface type='bridge'>
+            <mac address='52:54:00:1e:af:64'/>
+            <source bridge='br422442'/>
+            <model type='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+        </interface>
+        <console type='pty'>
+            <target type='serial' port='0'/>
+        </console>
+        <serial type='pty'>
+            <target port='0'/>
+        </serial>
+        
+        <hostdev mode="subsystem" type="pci" managed="yes">
+            <source>
+                <address domain="0" bus="1" slot="0" function="0"/>
+            </source>
+        </hostdev>
+
+        <hostdev mode="subsystem" type="pci" managed="yes">
+            <source>
+                <address domain="0" bus="2" slot="0" function="0"/>
+            </source>
+        </hostdev>
+    </devices>
+</domain>
\ No newline at end of file
diff --git a/crates/gpu-utils/x2 b/crates/gpu-utils/x2
new file mode 100644
index 0000000000..f9a4b36558
--- /dev/null
+++ b/crates/gpu-utils/x2
@@ -0,0 +1,45 @@
+<domain type='kvm'>
+    <name>test-id</name>
+    <memory unit='KiB'>8388608</memory>
+    <vcpu placement='static'>2</vcpu>
+    <cputune>
+        <vcpupin vcpu='0' cpuset='1'/>
+        <vcpupin vcpu='1' cpuset='8'/>
+    </cputune>
+    <os>
+        <type arch='x86_64' machine='pc-i440fx-2.9'>hvm</type>
+        <boot dev='hd'/>
+    </os>
+    <devices>
+        <disk type='file' device='disk'>
+            <driver name='qemu' type='qcow2'/>
+            <source file='test-image'/>
+            <target dev='vda' bus='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+        </disk>
+        <interface type='bridge'>
+            <mac address='52:54:00:1e:af:64'/>
+            <source bridge='br422442'/>
+            <model type='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+        </interface>
+        <console type='pty'>
+            <target type='serial' port='0'/>
+        </console>
+        <serial type='pty'>
+            <target port='0'/>
+        </serial>
+
+        <hostdev mode="subsystem" type="pci" managed="yes">
+            <source>
+                <address domain="0" bus="1" slot="0" function="0"/>
+            </source>
+        </hostdev>
+
+        <hostdev mode="subsystem" type="pci" managed="yes">
+            <source>
+                <address domain="0" bus="2" slot="0" function="0"/>
+            </source>
+        </hostdev>
+    </devices>
+</domain>
\ No newline at end of file
diff --git a/crates/vm-utils/Cargo.toml b/crates/vm-utils/Cargo.toml
index d5eefd2c85..c5bdd96430 100644
--- a/crates/vm-utils/Cargo.toml
+++ b/crates/vm-utils/Cargo.toml
@@ -9,6 +9,8 @@ thiserror.workspace = true
 tracing.workspace = true
 rand.workspace = true
 ccp-shared.workspace = true
+gpu-utils.workspace = true
+
 mac_address = "1.1.7"
 virt = "0.4.0"
 
diff --git a/crates/vm-utils/src/template.xml b/crates/vm-utils/src/template.xml
index a7ddbf3f2f..4d8b34c056 100644
--- a/crates/vm-utils/src/template.xml
+++ b/crates/vm-utils/src/template.xml
@@ -28,5 +28,6 @@
         <serial type='pty'>
             <target port='0'/>
         </serial>
+        {}
     </devices>
 </domain>
diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index 43674a865f..d2f957a902 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -1,4 +1,5 @@
 use ccp_shared::types::LogicalCoreId;
+use gpu_utils::PciLocation;
 use mac_address::MacAddress;
 use nonempty::NonEmpty;
 use rand::Rng;
@@ -101,6 +102,8 @@ pub enum VmError {
         #[source]
         err: virt::error::Error,
     },
+    #[error("Failed to get GPU PCI location: {0}")]
+    FailedToGetPCI(#[from] gpu_utils::PciError),
 }
 
 // The list of states is taken from the libvirt documentation
@@ -156,8 +159,10 @@ pub fn create_domain(uri: &str, params: &CreateVMDomainParams) -> Result<(), VmE
     match domain {
         None => {
             tracing::info!(target: "vm-utils","Domain with name {} doesn't exists. Creating", params.name);
+            // There's certainly better places to do this, but RN it doesn't really matter
+            let gpu_pci_locations = gpu_utils::get_gpu_pci()?;
             let mac = generate_random_mac();
-            let xml = prepare_xml(&params, mac.to_string().as_str());
+            let xml = prepare_xml(&params, mac.to_string().as_str(), &gpu_pci_locations);
             Domain::define_xml_flags(&conn, xml.as_str(), VIR_DOMAIN_DEFINE_VALIDATE)
                 .map_err(|err| VmError::FailedToCreateVMDomain { err })?;
         }
@@ -307,7 +312,26 @@ fn generate_random_mac() -> MacAddress {
     MacAddress::from(result)
 }
 
-fn prepare_xml(params: &CreateVMDomainParams, mac_address: &str) -> String {
+fn prepare_xml(
+    params: &CreateVMDomainParams,
+    mac_address: &str,
+    gpu_pci_location: &Vec<PciLocation>,
+) -> String {
+    fn prepare_pci_config(location: &PciLocation) -> String {
+        format!(
+            r#"
+        <hostdev mode="subsystem" type="pci" managed="yes">
+            <source>
+                <address domain="{domain}" bus="{bus}" slot="{slot}" function="{function}"/>
+            </source>
+        </hostdev>"#,
+            domain = location.segment(),
+            bus = location.bus(),
+            slot = location.device(),
+            function = location.function(),
+        )
+    }
+
     let mut mapping = String::new();
     for (index, logical_id) in params.cpus.iter().enumerate() {
         if index > 0 {
@@ -316,6 +340,12 @@ fn prepare_xml(params: &CreateVMDomainParams, mac_address: &str) -> String {
         mapping.push_str(format!("<vcpupin vcpu='{index}' cpuset='{logical_id}'/>").as_str());
     }
     let memory_in_kb = params.cpus.len() * 4 * 1024 * 1024; // 4Gbs per core
+    let gpu_pci_configuration = gpu_pci_location
+        .iter()
+        .map(prepare_pci_config)
+        .collect::<Vec<_>>()
+        .join("\n");
+
     format!(
         include_str!("template.xml"),
         params.name,
@@ -325,6 +355,7 @@ fn prepare_xml(params: &CreateVMDomainParams, mac_address: &str) -> String {
         params.image.display(),
         mac_address,
         params.bridge_name,
+        gpu_pci_configuration,
     )
 }
 
@@ -357,8 +388,27 @@ mod tests {
                 bridge_name: "br422442".to_string(),
             },
             "52:54:00:1e:af:64",
+            &vec![
+                PciLocation::with_bdf(1, 0, 0).unwrap(),
+                PciLocation::with_bdf(2, 0, 0).unwrap(),
+            ],
         );
-        assert_eq!(xml, include_str!("../tests/expected_vm_config.xml"))
+
+        // trim excessive whitespaces
+        let xml = xml
+            .lines()
+            .map(|line| line.trim())
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        let expected = include_str!("../tests/expected_vm_config.xml");
+        let expected = expected
+            .lines()
+            .map(|line| line.trim())
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        assert_eq!(xml, expected);
     }
 
     #[test]
diff --git a/crates/vm-utils/tests/expected_vm_config.xml b/crates/vm-utils/tests/expected_vm_config.xml
index c6df01f963..ff42cc1c58 100644
--- a/crates/vm-utils/tests/expected_vm_config.xml
+++ b/crates/vm-utils/tests/expected_vm_config.xml
@@ -29,5 +29,17 @@
         <serial type='pty'>
             <target port='0'/>
         </serial>
+
+        <hostdev mode="subsystem" type="pci" managed="yes">
+            <source>
+                <address domain="0" bus="1" slot="0" function="0"/>
+            </source>
+        </hostdev>
+
+        <hostdev mode="subsystem" type="pci" managed="yes">
+            <source>
+                <address domain="0" bus="2" slot="0" function="0"/>
+            </source>
+        </hostdev>
     </devices>
 </domain>

From ca227e700d4813af4be1d931a204cb82ac60208e Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 22 Aug 2024 16:46:19 +0200
Subject: [PATCH 08/32] fix lints

---
 crates/vm-utils/src/vm_utils.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index d2f957a902..403d40e2fe 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -162,7 +162,7 @@ pub fn create_domain(uri: &str, params: &CreateVMDomainParams) -> Result<(), VmE
             // There's certainly better places to do this, but RN it doesn't really matter
             let gpu_pci_locations = gpu_utils::get_gpu_pci()?;
             let mac = generate_random_mac();
-            let xml = prepare_xml(&params, mac.to_string().as_str(), &gpu_pci_locations);
+            let xml = prepare_xml(params, mac.to_string().as_str(), &gpu_pci_locations);
             Domain::define_xml_flags(&conn, xml.as_str(), VIR_DOMAIN_DEFINE_VALIDATE)
                 .map_err(|err| VmError::FailedToCreateVMDomain { err })?;
         }
@@ -315,7 +315,7 @@ fn generate_random_mac() -> MacAddress {
 fn prepare_xml(
     params: &CreateVMDomainParams,
     mac_address: &str,
-    gpu_pci_location: &Vec<PciLocation>,
+    gpu_pci_location: &[PciLocation],
 ) -> String {
     fn prepare_pci_config(location: &PciLocation) -> String {
         format!(
@@ -388,7 +388,7 @@ mod tests {
                 bridge_name: "br422442".to_string(),
             },
             "52:54:00:1e:af:64",
-            &vec![
+            &[
                 PciLocation::with_bdf(1, 0, 0).unwrap(),
                 PciLocation::with_bdf(2, 0, 0).unwrap(),
             ],

From ec2f624b74bd26ae995aaca49419dc14ee3d9c57 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Mon, 26 Aug 2024 13:18:40 +0200
Subject: [PATCH 09/32] modify the rules

---
 crates/vm-network-utils/src/lib.rs | 305 ++++++++++++++++++++++-------
 1 file changed, 239 insertions(+), 66 deletions(-)

diff --git a/crates/vm-network-utils/src/lib.rs b/crates/vm-network-utils/src/lib.rs
index 70e05ac6cb..d89a1a9ca5 100644
--- a/crates/vm-network-utils/src/lib.rs
+++ b/crates/vm-network-utils/src/lib.rs
@@ -52,52 +52,62 @@ pub fn clear_network(
     network_settings: &NetworkSettings,
     name: &str,
 ) -> Result<(), NetworkSetupError> {
-    clear_network_inner(network_settings, name).map_err(|err| NetworkSetupError::Clean {
-        chain_name: name.to_string(),
+    let ipt = iptables::new(false).map_err(|err| NetworkSetupError::Init {
         message: err.to_string(),
-    })
+    })?;
+    let rule_sets = vec![
+        fwd_rules(network_settings, name),
+        dnat_rules(network_settings, name),
+        snat_rules(network_settings, name),
+    ];
+    for rule_set in rule_sets {
+        clear_rules(&ipt, &rule_set)?;
+    }
+    Ok(())
 }
 
-fn clear_network_inner(
-    network_settings: &NetworkSettings,
-    name: &str,
-) -> Result<(), IpTablesError> {
-    let ipt = iptables::new(false)?;
-    {
-        let (chain_name, rules) = snat_rules(network_settings, name);
-        clear_chain(&ipt, &chain_name, &rules)?;
+fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupError> {
+    for insert_rules in &rules.insert_rules {
+        clear_existing_chain_rules(ipt, insert_rules).map_err(|err| NetworkSetupError::Clean {
+            chain_name: insert_rules.chain_name.clone(),
+            message: err.to_string(),
+        })?;
     }
 
-    {
-        let (chain_name, rules) = dnat_rules(network_settings, name);
-        clear_chain(&ipt, &chain_name, &rules)?;
+    for append_rules in &rules.append_rules {
+        clear_new_chain_rules(ipt, append_rules).map_err(|err| NetworkSetupError::Clean {
+            chain_name: append_rules.chain_name.clone(),
+            message: err.to_string(),
+        })?;
     }
 
-    {
-        let (chain_name, rules) = fwd_rules(network_settings, name);
-        clear_chain(&ipt, &chain_name, &rules)?;
-    }
     Ok(())
 }
 
-fn clear_chain(ipt: &IPTables, chain_name: &str, rules: &Vec<String>) -> Result<(), IpTablesError> {
-    let exists = ipt.chain_exists("nat", chain_name)?;
+fn clear_existing_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
+    let exists = ipt.chain_exists(rules.table_name, &rules.chain_name)?;
     if !exists {
-        tracing::warn!("Can't clean the chain {chain_name}: doesn't exist");
+        tracing::warn!("Can't clean the chain {}: doesn't exist", rules.chain_name);
         return Ok(());
     }
 
-    for rule in rules {
-        if ipt.exists("nat", chain_name, rule)? {
-            ipt.delete("nat", chain_name, rule)?;
+    for rule in &rules.rules {
+        if ipt.exists(rules.table_name, &rules.chain_name, rule)? {
+            ipt.delete(rules.table_name, &rules.chain_name, rule)?;
         } else {
             tracing::warn!(
-                "Can't clean a rule for the chain {chain_name}: doesn't exist; rule: {rule}"
+                "Can't clean a rule for the chain {chain_name}: doesn't exist; rule: {rule}",
+                chain_name = rules.chain_name,
             )
         }
     }
 
-    ipt.delete_chain("nat", chain_name)?;
+    Ok(())
+}
+
+fn clear_new_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
+    clear_existing_chain_rules(ipt, rules)?;
+    ipt.delete_chain(rules.table_name, &rules.chain_name)?;
     Ok(())
 }
 
@@ -106,8 +116,8 @@ fn setup_dnat(
     ipt: &IPTables,
     name: &str,
 ) -> Result<(), IpTablesError> {
-    let (chain_name, rules) = dnat_rules(network_settings, name);
-    apply_rules(ipt, &chain_name, &rules)?;
+    let rules = dnat_rules(network_settings, name);
+    add_rules(ipt, &rules)?;
     Ok(())
 }
 
@@ -116,8 +126,8 @@ fn setup_snat(
     ipt: &IPTables,
     name: &str,
 ) -> Result<(), IpTablesError> {
-    let (chain_name, rules) = snat_rules(network_settings, name);
-    apply_rules(ipt, &chain_name, &rules)?;
+    let rules = snat_rules(network_settings, name);
+    add_rules(ipt, &rules)?;
     Ok(())
 }
 
@@ -126,19 +136,42 @@ fn setup_fwd(
     ipt: &IPTables,
     name: &str,
 ) -> Result<(), IpTablesError> {
-    let (chain_name, rules) = fwd_rules(network_settings, name);
-    apply_rules(ipt, &chain_name, &rules)?;
+    let rules = fwd_rules(network_settings, name);
+    add_rules(ipt, &rules)?;
     Ok(())
 }
 
-fn apply_rules(ipt: &IPTables, name: &str, rules: &Vec<String>) -> Result<(), IpTablesError> {
-    ipt.new_chain("nat", name)?;
-    for rule in rules {
-        ipt.append("nat", name, rule)?;
+fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
+    for append_rules in &rules_set.append_rules {
+        ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
+        for r in &append_rules.rules {
+            ipt.append(append_rules.table_name, &append_rules.chain_name, r)?;
+        }
     }
+
+    for rule in &rules_set.insert_rules {
+        for r in &rule.rules {
+            // 1 is a default position when inserting a rule
+            ipt.insert(rule.table_name, &rule.chain_name, r, 1)?;
+        }
+    }
+
     Ok(())
 }
 
+#[derive(Debug)]
+struct RulesSet {
+    append_rules: Vec<IpTablesRules>,
+    insert_rules: Vec<IpTablesRules>,
+}
+
+#[derive(Debug)]
+struct IpTablesRules {
+    table_name: &'static str,
+    chain_name: String,
+    rules: Vec<String>,
+}
+
 // ```
 // iptables -t nat -N SNAT-${VM_NAME}
 // # Map the port range
@@ -152,7 +185,7 @@ fn apply_rules(ipt: &IPTables, name: &str, rules: &Vec<String>) -> Result<(), Ip
 //          -s ${VM_IP}
 //          -p tcp -m tcp
 //          --dport ${RNG_START}:${RNG_END}
-//          -d ${PUBLIC_IP}
+//          -d ${VM_IP}
 //          -j MASQUERADE
 // # Map SSH ports
 // iptables -A SNAT-${VM_NAME}
@@ -165,10 +198,10 @@ fn apply_rules(ipt: &IPTables, name: &str, rules: &Vec<String>) -> Result<(), Ip
 //          -s ${VM_IP}
 //          -p tcp -m tcp
 //          --dport ${MAP_HOST}
-//          -d ${PUBLIC_IP}
+//          -d ${VM_IP}
 //          -j MASQUERADE
 // ```
-fn snat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+fn snat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet {
     let public_ip = network_settings.public_ip;
     let vm_ip = network_settings.vm_ip;
     let vm_ssh_port = network_settings.vm_ssh_port;
@@ -178,28 +211,41 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
 
     let name = cut_chain_name(format!("SNAT-{name}"));
 
-    let ports_rules = format!(
+    let ports_rule = format!(
         "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j SNAT --to-source {public_ip}"
     );
 
-    let ssh_ports_rules =
+    let ssh_ports_rule =
         format!("-s {vm_ip} -p tcp -m tcp --dport {vm_ssh_port} -j SNAT --to-source {public_ip}");
 
-    let masquerade_ports_rules = format!(
-        "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -d {public_ip} -j MASQUERADE"
+    let masquerade_ports_rule = format!(
+        "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -d {vm_ip} -j MASQUERADE"
     );
 
-    let masquerade_ssh_ports_rules =
-        format!("-s {vm_ip} -p tcp -m tcp --dport {host_ssh_port} -d {public_ip} -j MASQUERADE");
-
-    let rules = vec![
-        ports_rules,
-        ssh_ports_rules,
-        masquerade_ports_rules,
-        masquerade_ssh_ports_rules,
-    ];
-
-    (name, rules)
+    let masquerade_ssh_ports_rule =
+        format!("-s {vm_ip} -p tcp -m tcp --dport {host_ssh_port} -d {vm_ip} -j MASQUERADE");
+
+    let append_rules = IpTablesRules {
+        table_name: "nat",
+        chain_name: name.clone(),
+        rules: vec![
+            ports_rule,
+            ssh_ports_rule,
+            masquerade_ports_rule,
+            masquerade_ssh_ports_rule,
+        ],
+    };
+
+    let postrouting_rule = IpTablesRules {
+        table_name: "nat",
+        chain_name: "POSTROUTING".to_string(),
+        rules: vec![format!("-s {vm_ip} -d {vm_ip} -j {name}")],
+    };
+
+    RulesSet {
+        append_rules: vec![append_rules],
+        insert_rules: vec![postrouting_rule],
+    }
 }
 
 // The corresponding iptables commands:
@@ -220,8 +266,17 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
 //          --dport ${MAP_HOST}
 //           --to-destination ${VM_IP}:${MAP_VM}
 //           -j DNAT
+//
+// iptables -t nat
+//          -I OUTPUT
+//          -d ${PUBLIC_IP}
+//          -j DNAT-${VM_NAME}
+// iptables -t nat
+//          -I PREROUTING
+//          -d ${PUBLIC_IP}
+//          -j DNAT-${VM_NAME}
 // ```
-fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet {
     let public_ip = network_settings.public_ip;
     let vm_ip = network_settings.vm_ip;
     let vm_ssh_port = network_settings.vm_ssh_port;
@@ -231,21 +286,41 @@ fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
 
     let name = cut_chain_name(format!("DNAT-{name}"));
 
-    let port_rules = format!(
+    let port_rule = format!(
         "-d {public_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j DNAT \
         --to-destination {vm_ip}:{port_start}-{port_end}",
     );
 
-    let ssh_port_rules = format!(
+    let ssh_port_rule = format!(
         "-d {public_ip} -p tcp -m tcp --dport {host_ssh_port} -j DNAT \
         --to-destination {vm_ip}:{vm_ssh_port}",
     );
-    let rules = vec![port_rules, ssh_port_rules];
-    (name, rules)
+    let append_rules = IpTablesRules {
+        table_name: "nat",
+        chain_name: name.clone(),
+        rules: vec![port_rule, ssh_port_rule],
+    };
+
+    let prerouting_rule = IpTablesRules {
+        table_name: "nat",
+        chain_name: "PREROUTING".to_string(),
+        rules: vec![format!("-d {public_ip} -j {name}")],
+    };
+
+    let output_rule = IpTablesRules {
+        table_name: "nat",
+        chain_name: "OUTPUT".to_string(),
+        rules: vec![format!("-d {public_ip} -j {name}")],
+    };
+
+    RulesSet {
+        append_rules: vec![append_rules],
+        insert_rules: vec![output_rule, prerouting_rule],
+    }
 }
 
 // ```
-// iptables -t nat -N FWD-${VM_NAME}
+// iptables -t filter -N FWD-${VM_NAME}
 // iptables -A FWD-${VM_NAME}
 //          -d ${VM_IP}
 //          -o ${BRIDGE_NAME}   # --out-interface
@@ -258,25 +333,45 @@ fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<St
 //          -p tcp -m tcp
 //          --dport ${MAP_VM}
 //          -j ACCEPT
+// iptables -t filter
+//          -I FORWARD
+//          -d {VM_IP}
+//          -j FWD-${VM_NAME}
 // ```
-fn fwd_rules(network_settings: &NetworkSettings, name: &str) -> (String, Vec<String>) {
+fn fwd_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet {
     let vm_ip = network_settings.vm_ip;
     let vm_ssh_port = network_settings.vm_ssh_port;
     let port_start = network_settings.port_range.0;
     let port_end = network_settings.port_range.1;
     let bridge_name = &network_settings.bridge_name;
 
-    let name = cut_chain_name(format!("FWD-{name}"));
+    let chain_name = cut_chain_name(format!("FWD-{name}"));
 
-    let port_rules = format!(
+    let port_rule = format!(
         "-d {vm_ip} -o {bridge_name} -p tcp -m tcp --dport {port_start}:{port_end} -j ACCEPT"
     );
 
-    let ssh_port_rules =
+    let ssh_port_rule =
         format!("-d {vm_ip} -o {bridge_name} -p tcp -m tcp --dport {vm_ssh_port} -j ACCEPT");
 
-    let rules = vec![port_rules, ssh_port_rules];
-    (name, rules)
+    let fwd_rule = format!("-d {vm_ip} -j {chain_name}");
+
+    let append_rules = IpTablesRules {
+        table_name: "filter",
+        chain_name,
+        rules: vec![port_rule, ssh_port_rule],
+    };
+
+    let insert_rules = IpTablesRules {
+        table_name: "filter",
+        chain_name: "FORWARD".to_string(),
+        rules: vec![fwd_rule],
+    };
+
+    RulesSet {
+        append_rules: vec![append_rules],
+        insert_rules: vec![insert_rules],
+    }
 }
 
 // iptables allows only 29 characters for the chain name
@@ -284,3 +379,81 @@ fn cut_chain_name(mut name: String) -> String {
     name.truncate(28);
     name
 }
+
+#[test]
+fn test() {
+    use std::str::FromStr;
+    fn to_string(r: &RulesSet) -> String {
+        let mut rules = Vec::new();
+        for a in &r.append_rules {
+            let fmt = format!(
+                "-t {table_name} -N {chain_name}",
+                table_name = a.table_name,
+                chain_name = a.chain_name
+            );
+            rules.push(fmt);
+            for rule in &a.rules {
+                let fmt = format!(
+                    "-t {table_name} -A {chain_name} {rule}",
+                    table_name = a.table_name,
+                    chain_name = a.chain_name
+                );
+                rules.push(fmt);
+            }
+        }
+
+        for i in &r.insert_rules {
+            for rule in &i.rules {
+                let fmt = format!(
+                    "-t {table_name} -I {chain_name} {rule}",
+                    table_name = i.table_name,
+                    chain_name = i.chain_name
+                );
+                rules.push(fmt);
+            }
+        }
+        rules.join("\n")
+    }
+
+    let ns = NetworkSettings {
+        public_ip: Ipv4Addr::from_str("1.1.1.1").unwrap(),
+        vm_ip: Ipv4Addr::from_str("2.2.2.2").unwrap(),
+        bridge_name: "br0".to_string(),
+        port_range: (1000, 65535),
+        host_ssh_port: 2222,
+        vm_ssh_port: 22,
+    };
+
+    {
+        let test = fwd_rules(&ns, "test");
+        let result = to_string(&test);
+        let expected = r#"-t filter -N FWD-test
+-t filter -A FWD-test -d 2.2.2.2 -o br0 -p tcp -m tcp --dport 1000:65535 -j ACCEPT
+-t filter -A FWD-test -d 2.2.2.2 -o br0 -p tcp -m tcp --dport 22 -j ACCEPT
+-t filter -I FORWARD -d 2.2.2.2 -j FWD-test"#;
+        assert_eq!(expected, result);
+    }
+
+    {
+        let test = dnat_rules(&ns, "test");
+        let result = to_string(&test);
+        let expected = r#"-t nat -N DNAT-test
+-t nat -A DNAT-test -d 1.1.1.1 -p tcp -m tcp --dport 1000:65535 -j DNAT --to-destination 2.2.2.2:1000-65535
+-t nat -A DNAT-test -d 1.1.1.1 -p tcp -m tcp --dport 2222 -j DNAT --to-destination 2.2.2.2:22
+-t nat -I OUTPUT -d 1.1.1.1 -j DNAT-test
+-t nat -I PREROUTING -d 1.1.1.1 -j DNAT-test"#;
+        assert_eq!(expected, result);
+    }
+
+    {
+        let test = snat_rules(&ns, "test");
+        let result = to_string(&test);
+        let expected = r#"-t nat -N SNAT-test
+-t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 1000:65535 -j SNAT --to-source 1.1.1.1
+-t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 22 -j SNAT --to-source 1.1.1.1
+-t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 1000:65535 -d 2.2.2.2 -j MASQUERADE
+-t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 2222 -d 2.2.2.2 -j MASQUERADE
+-t nat -I POSTROUTING -s 2.2.2.2 -d 2.2.2.2 -j SNAT-test"#;
+        assert_eq!(expected, result);
+    }
+}

From e327b56efc8409043223da36c1332fea6fd40121 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Mon, 26 Aug 2024 16:12:14 +0200
Subject: [PATCH 10/32] add gpu enable config

---
 crates/server-config/src/node_config.rs       |  2 +
 crates/server-config/src/resolved_config.rs   |  2 +
 crates/vm-utils/src/vm_utils.rs               | 45 ++++++++++++++++++-
 .../tests/expected_vm_config_without_gpu.xml  | 34 ++++++++++++++
 crates/workers/src/workers.rs                 |  5 ++-
 nox/src/node.rs                               | 17 ++++---
 6 files changed, 96 insertions(+), 9 deletions(-)
 create mode 100644 crates/vm-utils/tests/expected_vm_config_without_gpu.xml

diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index 80dcb962bd..70e9636eb0 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -624,6 +624,8 @@ fn default_dev_mode_config() -> DevModeConfig {
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct VmConfig {
     pub libvirt_uri: String,
+    #[serde(default)]
+    pub allow_gpu: bool,
     pub network: VmNetworkConfig,
 }
 
diff --git a/crates/server-config/src/resolved_config.rs b/crates/server-config/src/resolved_config.rs
index 79802bf617..1c93e6aaca 100644
--- a/crates/server-config/src/resolved_config.rs
+++ b/crates/server-config/src/resolved_config.rs
@@ -859,6 +859,7 @@ mod tests {
             r#"
             [vm]
             libvirt_uri = "qemu:///system"
+            allow_gpu = true
             [vm.network]
             bridge_name = "br422442"
             public_ip = "1.1.1.1"
@@ -880,6 +881,7 @@ mod tests {
                 config.node_config.vm,
                 Some(VmConfig {
                     libvirt_uri: "qemu:///system".to_string(),
+                    allow_gpu: true,
                     network: VmNetworkConfig {
                         bridge_name: "br422442".to_string(),
                         public_ip: Ipv4Addr::new(1, 1, 1, 1),
diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index 403d40e2fe..c6a43b044d 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -18,6 +18,7 @@ pub struct CreateVMDomainParams {
     image: PathBuf,
     cpus: NonEmpty<LogicalCoreId>,
     bridge_name: String,
+    allow_gpu: bool,
 }
 
 impl CreateVMDomainParams {
@@ -26,12 +27,14 @@ impl CreateVMDomainParams {
         image: PathBuf,
         cpus: NonEmpty<LogicalCoreId>,
         bridge_name: String,
+        allow_gpu: bool,
     ) -> Self {
         Self {
             name,
             image,
             cpus,
             bridge_name,
+            allow_gpu,
         }
     }
 }
@@ -160,7 +163,11 @@ pub fn create_domain(uri: &str, params: &CreateVMDomainParams) -> Result<(), VmE
         None => {
             tracing::info!(target: "vm-utils","Domain with name {} doesn't exists. Creating", params.name);
             // There's certainly better places to do this, but RN it doesn't really matter
-            let gpu_pci_locations = gpu_utils::get_gpu_pci()?;
+            let gpu_pci_locations = if params.allow_gpu {
+                gpu_utils::get_gpu_pci()?
+            } else {
+                vec![]
+            };
             let mac = generate_random_mac();
             let xml = prepare_xml(params, mac.to_string().as_str(), &gpu_pci_locations);
             Domain::define_xml_flags(&conn, xml.as_str(), VIR_DOMAIN_DEFINE_VALIDATE)
@@ -386,6 +393,7 @@ mod tests {
                 image: "test-image".into(),
                 cpus: nonempty![1.into(), 8.into()],
                 bridge_name: "br422442".to_string(),
+                allow_gpu: true,
             },
             "52:54:00:1e:af:64",
             &[
@@ -398,6 +406,7 @@ mod tests {
         let xml = xml
             .lines()
             .map(|line| line.trim())
+            .filter(|line| !line.is_empty())
             .collect::<Vec<_>>()
             .join("\n");
 
@@ -405,12 +414,45 @@ mod tests {
         let expected = expected
             .lines()
             .map(|line| line.trim())
+            .filter(|line| !line.is_empty())
             .collect::<Vec<_>>()
             .join("\n");
 
         assert_eq!(xml, expected);
     }
 
+    #[test]
+    fn test_prepare_xml_without_gpu() {
+        let xml = prepare_xml(
+            &CreateVMDomainParams {
+                name: "test-id".to_string(),
+                image: "test-image".into(),
+                cpus: nonempty![1.into(), 8.into()],
+                bridge_name: "br422442".to_string(),
+                allow_gpu: false,
+            },
+            "52:54:00:1e:af:64",
+            &[],
+        );
+
+        // trim excessive whitespaces
+        let xml = xml
+            .lines()
+            .map(|line| line.trim())
+            .filter(|line| !line.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        let expected = include_str!("../tests/expected_vm_config_without_gpu.xml");
+        let expected = expected
+            .lines()
+            .map(|line| line.trim())
+            .filter(|line| !line.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n");
+        assert_eq!(xml, expected);
+    }
+
     #[test]
     fn test_vm_creation() {
         log_utils::enable_logs();
@@ -429,6 +471,7 @@ mod tests {
                 image: image.clone(),
                 cpus: nonempty![1.into()],
                 bridge_name: "br422442".to_string(),
+                allow_gpu: false,
             },
         )
         .unwrap();
diff --git a/crates/vm-utils/tests/expected_vm_config_without_gpu.xml b/crates/vm-utils/tests/expected_vm_config_without_gpu.xml
new file mode 100644
index 0000000000..df2fd3df3b
--- /dev/null
+++ b/crates/vm-utils/tests/expected_vm_config_without_gpu.xml
@@ -0,0 +1,34 @@
+<domain type='kvm'>
+    <name>test-id</name>
+    <memory unit='KiB'>8388608</memory>
+    <vcpu placement='static'>2</vcpu>
+    <cputune>
+        <vcpupin vcpu='0' cpuset='1'/>
+        <vcpupin vcpu='1' cpuset='8'/>
+    </cputune>
+    <os>
+        <type arch='x86_64' machine='pc-i440fx-2.9'>hvm</type>
+        <boot dev='hd'/>
+    </os>
+    <devices>
+        <disk type='file' device='disk'>
+            <driver name='qemu' type='qcow2'/>
+            <source file='test-image'/>
+            <target dev='vda' bus='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+        </disk>
+        <interface type='bridge'>
+            <mac address='52:54:00:1e:af:64'/>
+            <source bridge='br422442'/>
+            <model type='virtio'/>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+        </interface>
+        <console type='pty'>
+            <target type='serial' port='0'/>
+        </console>
+        <serial type='pty'>
+            <target port='0'/>
+        </serial>
+
+    </devices>
+</domain>
diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index 0bb0eaf27f..a710944f02 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -123,13 +123,15 @@ impl WorkersConfig {
 pub struct VmConfig {
     /// Uri to the libvirt API
     libvirt_uri: String,
+    allow_gpu: bool,
     network: NetworkSettings,
 }
 
 impl VmConfig {
-    pub fn new(libvirt_uri: String, network: NetworkSettings) -> Self {
+    pub fn new(libvirt_uri: String, allow_gpu: bool, network: NetworkSettings) -> Self {
         Self {
             libvirt_uri,
+            allow_gpu,
             network,
         }
     }
@@ -705,6 +707,7 @@ impl Workers {
             worker_image,
             assignment,
             vm_config.network.bridge_name.clone(),
+            vm_config.allow_gpu,
         );
 
         vm_utils::create_domain(vm_config.libvirt_uri.clone().as_str(), &params)?;
diff --git a/nox/src/node.rs b/nox/src/node.rs
index 84ae4d3b92..a8cae9c15a 100644
--- a/nox/src/node.rs
+++ b/nox/src/node.rs
@@ -198,13 +198,16 @@ impl<RT: AquaRuntime> Node<RT> {
             key_storage.clone(),
         );
 
-        let workers_config =
-            WorkersConfig::new(
-                config.node_config.workers_queue_buffer,
-                config.node_config.vm.clone().map(|conf| {
-                    VmConfig::new(conf.libvirt_uri, to_vm_network_settings(conf.network))
-                }),
-            );
+        let workers_config = WorkersConfig::new(
+            config.node_config.workers_queue_buffer,
+            config.node_config.vm.clone().map(|conf| {
+                VmConfig::new(
+                    conf.libvirt_uri,
+                    conf.allow_gpu,
+                    to_vm_network_settings(conf.network),
+                )
+            }),
+        );
 
         let (workers, worker_events) = Workers::from_path(
             workers_config,

From d11e7d8013aa75129cccb420660190fd9130172b Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Tue, 27 Aug 2024 16:29:39 +0200
Subject: [PATCH 11/32] fix remove

---
 crates/gpu-utils/x1            | 45 ----------------------------------
 crates/gpu-utils/x2            | 45 ----------------------------------
 crates/workers/src/workers.rs  |  6 +++--
 sorcerer/src/worker_builins.rs |  5 ++--
 4 files changed, 7 insertions(+), 94 deletions(-)
 delete mode 100644 crates/gpu-utils/x1
 delete mode 100644 crates/gpu-utils/x2

diff --git a/crates/gpu-utils/x1 b/crates/gpu-utils/x1
deleted file mode 100644
index 562d708066..0000000000
--- a/crates/gpu-utils/x1
+++ /dev/null
@@ -1,45 +0,0 @@
-<domain type='kvm'>
-    <name>test-id</name>
-    <memory unit='KiB'>8388608</memory>
-    <vcpu placement='static'>2</vcpu>
-    <cputune>
-        <vcpupin vcpu='0' cpuset='1'/>
-        <vcpupin vcpu='1' cpuset='8'/>
-    </cputune>
-    <os>
-        <type arch='x86_64' machine='pc-i440fx-2.9'>hvm</type>
-        <boot dev='hd'/>
-    </os>
-    <devices>
-        <disk type='file' device='disk'>
-            <driver name='qemu' type='qcow2'/>
-            <source file='test-image'/>
-            <target dev='vda' bus='virtio'/>
-            <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
-        </disk>
-        <interface type='bridge'>
-            <mac address='52:54:00:1e:af:64'/>
-            <source bridge='br422442'/>
-            <model type='virtio'/>
-            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
-        </interface>
-        <console type='pty'>
-            <target type='serial' port='0'/>
-        </console>
-        <serial type='pty'>
-            <target port='0'/>
-        </serial>
-        
-        <hostdev mode="subsystem" type="pci" managed="yes">
-            <source>
-                <address domain="0" bus="1" slot="0" function="0"/>
-            </source>
-        </hostdev>
-
-        <hostdev mode="subsystem" type="pci" managed="yes">
-            <source>
-                <address domain="0" bus="2" slot="0" function="0"/>
-            </source>
-        </hostdev>
-    </devices>
-</domain>
\ No newline at end of file
diff --git a/crates/gpu-utils/x2 b/crates/gpu-utils/x2
deleted file mode 100644
index f9a4b36558..0000000000
--- a/crates/gpu-utils/x2
+++ /dev/null
@@ -1,45 +0,0 @@
-<domain type='kvm'>
-    <name>test-id</name>
-    <memory unit='KiB'>8388608</memory>
-    <vcpu placement='static'>2</vcpu>
-    <cputune>
-        <vcpupin vcpu='0' cpuset='1'/>
-        <vcpupin vcpu='1' cpuset='8'/>
-    </cputune>
-    <os>
-        <type arch='x86_64' machine='pc-i440fx-2.9'>hvm</type>
-        <boot dev='hd'/>
-    </os>
-    <devices>
-        <disk type='file' device='disk'>
-            <driver name='qemu' type='qcow2'/>
-            <source file='test-image'/>
-            <target dev='vda' bus='virtio'/>
-            <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
-        </disk>
-        <interface type='bridge'>
-            <mac address='52:54:00:1e:af:64'/>
-            <source bridge='br422442'/>
-            <model type='virtio'/>
-            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
-        </interface>
-        <console type='pty'>
-            <target type='serial' port='0'/>
-        </console>
-        <serial type='pty'>
-            <target port='0'/>
-        </serial>
-
-        <hostdev mode="subsystem" type="pci" managed="yes">
-            <source>
-                <address domain="0" bus="1" slot="0" function="0"/>
-            </source>
-        </hostdev>
-
-        <hostdev mode="subsystem" type="pci" managed="yes">
-            <source>
-                <address domain="0" bus="2" slot="0" function="0"/>
-            </source>
-        </hostdev>
-    </devices>
-</domain>
\ No newline at end of file
diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index a710944f02..cf5f9b05cc 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -379,7 +379,11 @@ impl Workers {
             .send(Event::WorkerRemoved { worker_id })
             .await
             .map_err(|_err| WorkersError::FailedToNotifySubsystem { worker_id })?;
+
+        self.remove_vm(worker_id)?;
+
         remove_worker(&self.workers_dir, worker_id).await?;
+
         self.key_storage
             .remove_key_pair(worker_id)
             .await
@@ -395,8 +399,6 @@ impl Workers {
             let removed_runtime = runtimes.remove(&worker_id);
             let removed_assignments = assignments.remove(&worker_id);
 
-            self.remove_vm(worker_id)?;
-
             debug_assert!(removed_worker_id.is_some(), "worker_id does not exist");
             debug_assert!(removed_worker_info.is_some(), "worker info does not exist");
             debug_assert!(removed_runtime.is_some(), "worker runtime does not exist");
diff --git a/sorcerer/src/worker_builins.rs b/sorcerer/src/worker_builins.rs
index 41f17ea84e..5607680672 100644
--- a/sorcerer/src/worker_builins.rs
+++ b/sorcerer/src/worker_builins.rs
@@ -64,7 +64,6 @@ pub(crate) async fn create_worker(
 pub(crate) fn get_worker_peer_id(args: Args, workers: Arc<Workers>) -> Result<JValue, JError> {
     let mut args = args.function_args.into_iter();
     let deal_id: String = Args::next("deal_id", &mut args)?;
-
     Ok(JValue::Array(
         workers
             .get_worker_id(deal_id.into())
@@ -99,7 +98,7 @@ pub(crate) async fn remove_worker(
             {
                 return Err(JError::new(format!("Worker {worker_id} can be removed only by worker creator {worker_creator}, host or a host manager")));
             }
-            workers.remove_worker(worker_id).await?;
+
             let spells: Vec<_> = spell_storage.get_registered_spells_by(peer_scope);
             for s in spells {
                 remove_spell(
@@ -118,7 +117,9 @@ pub(crate) async fn remove_worker(
                 })
                 .await?;
             }
+
             services.remove_services(peer_scope).await?;
+            workers.remove_worker(worker_id).await?;
         }
         PeerScope::Host => return Err(JError::new(format!("Worker {worker_id} can be removed"))),
     };

From 28f075cd8af363b5c7aa4b5c0fc1779b6c4eebca Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Tue, 27 Aug 2024 17:42:30 +0200
Subject: [PATCH 12/32] add IOMMU groups

---
 Cargo.lock                          |  1 +
 crates/gpu-utils/Cargo.toml         |  3 +-
 crates/gpu-utils/src/lib.rs         | 77 +++++++++++++++++++++++++++--
 crates/vm-network-utils/Cargo.toml  |  4 ++
 crates/vm-network-utils/bin/main.rs | 18 +++++++
 5 files changed, 99 insertions(+), 4 deletions(-)
 create mode 100644 crates/vm-network-utils/bin/main.rs

diff --git a/Cargo.lock b/Cargo.lock
index 0dc9bec821..96e525b62d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3137,6 +3137,7 @@ version = "0.1.0"
 dependencies = [
  "pci-info",
  "thiserror",
+ "tracing",
 ]
 
 [[package]]
diff --git a/crates/gpu-utils/Cargo.toml b/crates/gpu-utils/Cargo.toml
index 03cba6d18f..27e39854e4 100644
--- a/crates/gpu-utils/Cargo.toml
+++ b/crates/gpu-utils/Cargo.toml
@@ -7,4 +7,5 @@ edition = "2021"
 
 [dependencies]
 pci-info = "0.1.0"
-thiserror = { workspace = true }
\ No newline at end of file
+thiserror = { workspace = true }
+tracing = "0.1.40"
\ No newline at end of file
diff --git a/crates/gpu-utils/src/lib.rs b/crates/gpu-utils/src/lib.rs
index 50cf674191..556fc570ba 100644
--- a/crates/gpu-utils/src/lib.rs
+++ b/crates/gpu-utils/src/lib.rs
@@ -1,5 +1,6 @@
 use pci_info::pci_enums::PciDeviceClass::DisplayController;
 use pci_info::{PciDeviceEnumerationError, PciInfo, PciInfoError, PciInfoPropertyError};
+use std::collections::{HashMap, HashSet};
 use thiserror::Error;
 
 pub use pci_info::PciLocation;
@@ -18,15 +19,42 @@ pub enum PciError {
 
 pub fn get_gpu_pci() -> Result<Vec<PciLocation>, PciError> {
     let info = PciInfo::enumerate_pci()?;
-    let mut result = Vec::new();
+    let mut gpu_devices = Vec::new();
     for device in info {
         let device = device?;
         let device_class = process_property_result(device.device_class())?;
         if device_class == DisplayController {
-            result.push(process_property_result(device.location())?);
+            gpu_devices.push(process_property_result(device.location())?);
+        }
+    }
+
+    match get_iommu_groups() {
+        Ok(iommu_groups) => {
+            let result = iommu_groups
+                .iter()
+                .flat_map(|(_, devices)| {
+                    gpu_devices
+                        .iter()
+                        .filter_map(|gpu_device| {
+                            if devices.contains(gpu_device) {
+                                Some(devices.clone())
+                            } else {
+                                None
+                            }
+                        })
+                        .flatten()
+                        .collect::<HashSet<_>>()
+                })
+                .collect::<Vec<_>>();
+            Ok(result)
+        }
+        Err(err) => {
+            tracing::warn!(
+                "Couldn't get IOMMU groups: {err}. Ignoring groups, provide list of PCI nevertheless: {gpu_devices:?}",
+            );
+            Ok(gpu_devices)
         }
     }
-    Ok(result)
 }
 
 fn process_property_result<T>(result: Result<T, &PciInfoPropertyError>) -> Result<T, PciError> {
@@ -36,3 +64,46 @@ fn process_property_result<T>(result: Result<T, &PciInfoPropertyError>) -> Resul
         Err(PciInfoPropertyError::Error(err)) => Err(PciError::PciInfoPropertyError(err.clone())),
     }
 }
+
+type IommuGroup = HashMap<String, HashSet<PciLocation>>;
+
+fn get_iommu_groups() -> Result<IommuGroup, std::io::Error> {
+    const IOMMU_GROUP_PATH: &str = "/sys/kernel/iommu_groups";
+    let mut devices_by_group = HashMap::new();
+
+    for iommu_group_result in std::fs::read_dir(IOMMU_GROUP_PATH)? {
+        if let Ok(iommu_group) = iommu_group_result {
+            let group_name = iommu_group.file_name().to_string_lossy().to_string();
+
+            let path = iommu_group.path().join("devices");
+
+            let group = std::fs::read_dir(path)?
+                .flatten()
+                .filter_map(|device| {
+                    parse_pci_location(device.file_name().to_string_lossy().to_string())
+                })
+                .collect::<HashSet<_>>();
+
+            devices_by_group.insert(group_name, group);
+        } else {
+            tracing::warn!("cannot get an IOMMU group: {:?}", iommu_group_result);
+        }
+    }
+
+    Ok(devices_by_group)
+}
+
+// Location format:
+// <segment>:<bus>:<device>.<function>: 0000:00:00.0
+fn parse_pci_location(location: String) -> Option<PciLocation> {
+    let parts: Vec<&str> = location.split(':').collect();
+    if parts.len() != 3 {
+        return None;
+    }
+    // the segment part is always 0
+    let bus = u8::from_str_radix(parts[1], 16).ok()?;
+    let device = u8::from_str_radix(parts[2].split('.').next()?, 16).ok()?;
+    let function = u8::from_str_radix(parts[2].split('.').last()?, 16).ok()?;
+
+    PciLocation::with_bdf(bus, device, function).ok()
+}
diff --git a/crates/vm-network-utils/Cargo.toml b/crates/vm-network-utils/Cargo.toml
index 38231f476d..a4327bdef5 100644
--- a/crates/vm-network-utils/Cargo.toml
+++ b/crates/vm-network-utils/Cargo.toml
@@ -3,6 +3,10 @@ name = "vm-network-utils"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = 'vm-network-utils'
+path = "bin/main.rs"
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 iptables = "0.5.2"
diff --git a/crates/vm-network-utils/bin/main.rs b/crates/vm-network-utils/bin/main.rs
new file mode 100644
index 0000000000..318ae9aad2
--- /dev/null
+++ b/crates/vm-network-utils/bin/main.rs
@@ -0,0 +1,18 @@
+use std::net::Ipv4Addr;
+use std::str::FromStr;
+use vm_network_utils::{clear_network, setup_network, NetworkSettings};
+
+// bin for local tests
+fn main() {
+    let ns = NetworkSettings {
+        public_ip: Ipv4Addr::from_str("1.1.1.1").unwrap(),
+        vm_ip: Ipv4Addr::from_str("2.2.2.2").unwrap(),
+        bridge_name: "br0".to_string(),
+        port_range: (1000, 65535),
+        host_ssh_port: 2222,
+        vm_ssh_port: 22,
+    };
+    //let result = setup_network(&ns, "test");
+    let result = clear_network(&ns, "12D3KooWAb7dquiiyrxZEchx");
+    println!("{result:?}");
+}

From c007bca2d2a0a0d598da49c57f4ac7751ebe2379 Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 12:30:29 +0300
Subject: [PATCH 13/32] fix(config): add default value for
 VmNetworkConfig::vm_ip

---
 crates/server-config/src/node_config.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index 80dcb962bd..ff4337ee75 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -631,6 +631,7 @@ pub struct VmConfig {
 pub struct VmNetworkConfig {
     pub bridge_name: String,
     pub public_ip: Ipv4Addr,
+    #[serde(default = "default_vm_ip")]
     pub vm_ip: Ipv4Addr,
     #[serde(default = "default_host_ssh_port")]
     pub host_ssh_port: u16,
@@ -648,6 +649,11 @@ fn default_vm_ssh_port() -> u16 {
     22
 }
 
+fn default_vm_ip() -> &'static str {
+    "192.168.122.112"
+}
+
+
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct PortRangeConfig {
     pub start: u16,

From 3e235457050c4a0ff592aaa3721857cc1bfe121b Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 12:45:13 +0300
Subject: [PATCH 14/32] fix(config): fix type error

---
 crates/server-config/src/node_config.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index ff4337ee75..095f46d132 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -649,8 +649,8 @@ fn default_vm_ssh_port() -> u16 {
     22
 }
 
-fn default_vm_ip() -> &'static str {
-    "192.168.122.112"
+fn default_vm_ip() -> Ipv4Addr {
+    Ipv4Addr::new(192, 168, 122, 112)
 }
 
 

From 33d341bb9264c1b6795d9886af8dfc39b42bda7b Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Wed, 28 Aug 2024 12:01:53 +0200
Subject: [PATCH 15/32] do not export non-endpoint pci devices

---
 crates/gpu-utils/src/lib.rs     | 42 ++++++++++++++++++++-------------
 crates/vm-utils/src/vm_utils.rs |  2 +-
 2 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/crates/gpu-utils/src/lib.rs b/crates/gpu-utils/src/lib.rs
index 556fc570ba..cc61816536 100644
--- a/crates/gpu-utils/src/lib.rs
+++ b/crates/gpu-utils/src/lib.rs
@@ -1,4 +1,5 @@
-use pci_info::pci_enums::PciDeviceClass::DisplayController;
+use pci_info::pci_enums::PciDeviceClass;
+use pci_info::pci_enums::PciDeviceClass::{Bridge, DisplayController};
 use pci_info::{PciDeviceEnumerationError, PciInfo, PciInfoError, PciInfoPropertyError};
 use std::collections::{HashMap, HashSet};
 use thiserror::Error;
@@ -17,35 +18,38 @@ pub enum PciError {
     UnsupportedProperty,
 }
 
-pub fn get_gpu_pci() -> Result<Vec<PciLocation>, PciError> {
+pub fn get_gpu_pci() -> Result<HashSet<PciLocation>, PciError> {
     let info = PciInfo::enumerate_pci()?;
-    let mut gpu_devices = Vec::new();
+    // List of GPU devices
+    let mut gpu_devices = HashSet::new();
+    // Map of all PCI devices with their classes, for IOMMU groups processing
+    let mut pci_devices = HashMap::new();
+
     for device in info {
         let device = device?;
         let device_class = process_property_result(device.device_class())?;
+        let device_location = process_property_result(device.location())?;
         if device_class == DisplayController {
-            gpu_devices.push(process_property_result(device.location())?);
+            gpu_devices.insert(device_location);
         }
+        pci_devices.insert(device_location, device_class);
     }
 
     match get_iommu_groups() {
         Ok(iommu_groups) => {
+            // Find all devices that are in the same IOMMU group as the GPU devices
             let result = iommu_groups
-                .iter()
-                .flat_map(|(_, devices)| {
+                .into_iter()
+                .filter(|(_, devices)| {
+                    // Find if this IOMMU groups contains a GPU device
                     gpu_devices
                         .iter()
-                        .filter_map(|gpu_device| {
-                            if devices.contains(gpu_device) {
-                                Some(devices.clone())
-                            } else {
-                                None
-                            }
-                        })
-                        .flatten()
-                        .collect::<HashSet<_>>()
+                        .any(|gpu_device| devices.contains(gpu_device))
                 })
-                .collect::<Vec<_>>();
+                .flat_map(|(_, devices)| devices)
+                // We want to filter non-endpoint devies
+                .filter(|device| pci_devices.get(device).map_or(false, is_endpoint_device))
+                .collect::<HashSet<_>>();
             Ok(result)
         }
         Err(err) => {
@@ -57,6 +61,12 @@ pub fn get_gpu_pci() -> Result<Vec<PciLocation>, PciError> {
     }
 }
 
+// AFAIK the bridge devices are the only non-endpoint devices
+// May require to update this function if there are other non-endpoint devices
+fn is_endpoint_device(device_class: &PciDeviceClass) -> bool {
+    *device_class != Bridge
+}
+
 fn process_property_result<T>(result: Result<T, &PciInfoPropertyError>) -> Result<T, PciError> {
     match result {
         Ok(device) => Ok(device),
diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index c6a43b044d..c6c7670d52 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -164,7 +164,7 @@ pub fn create_domain(uri: &str, params: &CreateVMDomainParams) -> Result<(), VmE
             tracing::info!(target: "vm-utils","Domain with name {} doesn't exists. Creating", params.name);
             // There's certainly better places to do this, but RN it doesn't really matter
             let gpu_pci_locations = if params.allow_gpu {
-                gpu_utils::get_gpu_pci()?
+                gpu_utils::get_gpu_pci()?.into_iter().collect::<Vec<_>>()
             } else {
                 vec![]
             };

From 01bbe249ea1caee11411e732931c0f7fedff13f9 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Wed, 28 Aug 2024 12:30:01 +0200
Subject: [PATCH 16/32] fix

---
 crates/gpu-utils/src/lib.rs | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/crates/gpu-utils/src/lib.rs b/crates/gpu-utils/src/lib.rs
index cc61816536..b0eea235af 100644
--- a/crates/gpu-utils/src/lib.rs
+++ b/crates/gpu-utils/src/lib.rs
@@ -35,30 +35,26 @@ pub fn get_gpu_pci() -> Result<HashSet<PciLocation>, PciError> {
         pci_devices.insert(device_location, device_class);
     }
 
-    match get_iommu_groups() {
+    let result = match get_iommu_groups() {
         Ok(iommu_groups) => {
             // Find all devices that are in the same IOMMU group as the GPU devices
-            let result = iommu_groups
+            iommu_groups
                 .into_iter()
-                .filter(|(_, devices)| {
-                    // Find if this IOMMU groups contains a GPU device
-                    gpu_devices
-                        .iter()
-                        .any(|gpu_device| devices.contains(gpu_device))
-                })
+                // Find if this IOMMU groups contains a GPU device
+                .filter(|(_, devices)| gpu_devices.intersection(devices).next().is_some())
                 .flat_map(|(_, devices)| devices)
-                // We want to filter non-endpoint devies
+                // We want to filter non-endpoint devices
                 .filter(|device| pci_devices.get(device).map_or(false, is_endpoint_device))
-                .collect::<HashSet<_>>();
-            Ok(result)
+                .collect::<HashSet<_>>()
         }
         Err(err) => {
             tracing::warn!(
                 "Couldn't get IOMMU groups: {err}. Ignoring groups, provide list of PCI nevertheless: {gpu_devices:?}",
             );
-            Ok(gpu_devices)
+            gpu_devices
         }
-    }
+    };
+    Ok(result)
 }
 
 // AFAIK the bridge devices are the only non-endpoint devices

From b768a7f5b51ce94ac0d91751e5b02ece258150d3 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Wed, 28 Aug 2024 12:53:39 +0200
Subject: [PATCH 17/32] fix

---
 crates/vm-network-utils/bin/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/crates/vm-network-utils/bin/main.rs b/crates/vm-network-utils/bin/main.rs
index 318ae9aad2..fe6c5dc4f3 100644
--- a/crates/vm-network-utils/bin/main.rs
+++ b/crates/vm-network-utils/bin/main.rs
@@ -1,6 +1,7 @@
 use std::net::Ipv4Addr;
 use std::str::FromStr;
-use vm_network_utils::{clear_network, setup_network, NetworkSettings};
+
+use vm_network_utils::{clear_network, NetworkSettings};
 
 // bin for local tests
 fn main() {

From cc42e1defa2aeb4cc8e4b494b497607c787db505 Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 17:13:14 +0300
Subject: [PATCH 18/32] fix: default values for all VmConfig except public_ip,
 move linux-specific API to a separate mod

---
 crates/server-config/src/node_config.rs       |  11 +-
 crates/vm-network-utils/Cargo.toml            |   4 +-
 crates/vm-network-utils/bin/main.rs           |  20 ++-
 crates/vm-network-utils/src/lib.rs            | 130 ++++--------------
 crates/vm-network-utils/src/linux.rs          | 127 +++++++++++++++++
 .../vm-network-utils/src/non_linux_mocks.rs   |  36 +++++
 6 files changed, 223 insertions(+), 105 deletions(-)
 create mode 100644 crates/vm-network-utils/src/linux.rs
 create mode 100644 crates/vm-network-utils/src/non_linux_mocks.rs

diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index 6c61bf40af..f13b4e18ce 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -623,6 +623,7 @@ fn default_dev_mode_config() -> DevModeConfig {
 
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct VmConfig {
+    #[serde(default = "default_libvirt_uri")]
     pub libvirt_uri: String,
     #[serde(default)]
     pub allow_gpu: bool,
@@ -631,6 +632,7 @@ pub struct VmConfig {
 
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct VmNetworkConfig {
+    #[serde(default = "default_bridge_name")]
     pub bridge_name: String,
     pub public_ip: Ipv4Addr,
     #[serde(default = "default_vm_ip")]
@@ -643,6 +645,14 @@ pub struct VmNetworkConfig {
     pub port_range: PortRangeConfig,
 }
 
+fn default_libvirt_uri() -> String {
+    String::from("qemu:///system")
+}
+
+fn default_bridge_name() -> String {
+    String::from("br0")
+}
+
 fn default_host_ssh_port() -> u16 {
     2222
 }
@@ -655,7 +665,6 @@ fn default_vm_ip() -> Ipv4Addr {
     Ipv4Addr::new(192, 168, 122, 112)
 }
 
-
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct PortRangeConfig {
     pub start: u16,
diff --git a/crates/vm-network-utils/Cargo.toml b/crates/vm-network-utils/Cargo.toml
index a4327bdef5..6ac110882b 100644
--- a/crates/vm-network-utils/Cargo.toml
+++ b/crates/vm-network-utils/Cargo.toml
@@ -7,9 +7,9 @@ edition = "2021"
 name = 'vm-network-utils'
 path = "bin/main.rs"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-[dependencies]
+[target.'cfg(target_os = "linux")'.dependencies]
 iptables = "0.5.2"
 
+[dependencies]
 thiserror = { workspace = true }
 tracing = { workspace = true }
\ No newline at end of file
diff --git a/crates/vm-network-utils/bin/main.rs b/crates/vm-network-utils/bin/main.rs
index fe6c5dc4f3..18b1e733fe 100644
--- a/crates/vm-network-utils/bin/main.rs
+++ b/crates/vm-network-utils/bin/main.rs
@@ -1,6 +1,24 @@
+/*
+ * Nox Fluence Peer
+ *
+ * Copyright (C) 2024 Fluence DAO
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 use std::net::Ipv4Addr;
 use std::str::FromStr;
-
 use vm_network_utils::{clear_network, NetworkSettings};
 
 // bin for local tests
diff --git a/crates/vm-network-utils/src/lib.rs b/crates/vm-network-utils/src/lib.rs
index d89a1a9ca5..1bd332b34a 100644
--- a/crates/vm-network-utils/src/lib.rs
+++ b/crates/vm-network-utils/src/lib.rs
@@ -1,7 +1,35 @@
-use iptables::IPTables;
+/*
+ * Nox Fluence Peer
+ *
+ * Copyright (C) 2024 Fluence DAO
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
 use std::net::Ipv4Addr;
+
 use thiserror::Error;
 
+/// IPTables is a Linux-specific tool, so API is Linux-specific as well.
+/// Here we account for that, importing valid APIs on Linux, and a set of no-op mocks for any other OS.
+#[cfg(target_os = "linux")]
+pub use linux::*;
+#[cfg(not(target_os = "linux"))]
+pub use non_linux_mocks::*;
+
+mod linux;
+mod non_linux_mocks;
+
 pub struct NetworkSettings {
     pub public_ip: Ipv4Addr,
     pub vm_ip: Ipv4Addr,
@@ -29,88 +57,6 @@ pub enum NetworkSetupError {
 
 type IpTablesError = Box<dyn std::error::Error>;
 
-pub fn setup_network(
-    network_settings: &NetworkSettings,
-    name: &str,
-) -> Result<(), NetworkSetupError> {
-    let ipt = iptables::new(false).map_err(|err| NetworkSetupError::Init {
-        message: err.to_string(),
-    })?;
-    setup_snat(network_settings, &ipt, name).map_err(|err| NetworkSetupError::SNat {
-        message: err.to_string(),
-    })?;
-    setup_dnat(network_settings, &ipt, name).map_err(|err| NetworkSetupError::DNat {
-        message: err.to_string(),
-    })?;
-    setup_fwd(network_settings, &ipt, name).map_err(|err| NetworkSetupError::Fwd {
-        message: err.to_string(),
-    })?;
-    Ok(())
-}
-
-pub fn clear_network(
-    network_settings: &NetworkSettings,
-    name: &str,
-) -> Result<(), NetworkSetupError> {
-    let ipt = iptables::new(false).map_err(|err| NetworkSetupError::Init {
-        message: err.to_string(),
-    })?;
-    let rule_sets = vec![
-        fwd_rules(network_settings, name),
-        dnat_rules(network_settings, name),
-        snat_rules(network_settings, name),
-    ];
-    for rule_set in rule_sets {
-        clear_rules(&ipt, &rule_set)?;
-    }
-    Ok(())
-}
-
-fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupError> {
-    for insert_rules in &rules.insert_rules {
-        clear_existing_chain_rules(ipt, insert_rules).map_err(|err| NetworkSetupError::Clean {
-            chain_name: insert_rules.chain_name.clone(),
-            message: err.to_string(),
-        })?;
-    }
-
-    for append_rules in &rules.append_rules {
-        clear_new_chain_rules(ipt, append_rules).map_err(|err| NetworkSetupError::Clean {
-            chain_name: append_rules.chain_name.clone(),
-            message: err.to_string(),
-        })?;
-    }
-
-    Ok(())
-}
-
-fn clear_existing_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
-    let exists = ipt.chain_exists(rules.table_name, &rules.chain_name)?;
-    if !exists {
-        tracing::warn!("Can't clean the chain {}: doesn't exist", rules.chain_name);
-        return Ok(());
-    }
-
-    for rule in &rules.rules {
-        if ipt.exists(rules.table_name, &rules.chain_name, rule)? {
-            ipt.delete(rules.table_name, &rules.chain_name, rule)?;
-        } else {
-            tracing::warn!(
-                "Can't clean a rule for the chain {chain_name}: doesn't exist; rule: {rule}",
-                chain_name = rules.chain_name,
-            )
-        }
-    }
-
-    Ok(())
-}
-
-fn clear_new_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
-    clear_existing_chain_rules(ipt, rules)?;
-    ipt.delete_chain(rules.table_name, &rules.chain_name)?;
-    Ok(())
-}
-
 fn setup_dnat(
     network_settings: &NetworkSettings,
     ipt: &IPTables,
@@ -141,24 +87,6 @@ fn setup_fwd(
     Ok(())
 }
 
-fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
-    for append_rules in &rules_set.append_rules {
-        ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
-        for r in &append_rules.rules {
-            ipt.append(append_rules.table_name, &append_rules.chain_name, r)?;
-        }
-    }
-
-    for rule in &rules_set.insert_rules {
-        for r in &rule.rules {
-            // 1 is a default position when inserting a rule
-            ipt.insert(rule.table_name, &rule.chain_name, r, 1)?;
-        }
-    }
-
-    Ok(())
-}
-
 #[derive(Debug)]
 struct RulesSet {
     append_rules: Vec<IpTablesRules>,
diff --git a/crates/vm-network-utils/src/linux.rs b/crates/vm-network-utils/src/linux.rs
new file mode 100644
index 0000000000..d4cb8bb8f9
--- /dev/null
+++ b/crates/vm-network-utils/src/linux.rs
@@ -0,0 +1,127 @@
+/*
+ * Nox Fluence Peer
+ *
+ * Copyright (C) 2024 Fluence DAO
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#[cfg(target_os = "linux")]
+use iptables::IPTables;
+
+#[cfg(target_os = "linux")]
+pub fn setup_network(
+    network_settings: &NetworkSettings,
+    name: &str,
+) -> Result<(), NetworkSetupError> {
+    let ipt = iptables::new(false).map_err(|err| NetworkSetupError::Init {
+        message: err.to_string(),
+    })?;
+    setup_snat(network_settings, &ipt, name).map_err(|err| NetworkSetupError::SNat {
+        message: err.to_string(),
+    })?;
+    setup_dnat(network_settings, &ipt, name).map_err(|err| NetworkSetupError::DNat {
+        message: err.to_string(),
+    })?;
+    setup_fwd(network_settings, &ipt, name).map_err(|err| NetworkSetupError::Fwd {
+        message: err.to_string(),
+    })?;
+    Ok(())
+}
+
+#[cfg(target_os = "linux")]
+pub fn clear_network(
+    network_settings: &NetworkSettings,
+    name: &str,
+) -> Result<(), NetworkSetupError> {
+    let ipt = iptables::new(false).map_err(|err| NetworkSetupError::Init {
+        message: err.to_string(),
+    })?;
+    let rule_sets = vec![
+        fwd_rules(network_settings, name),
+        dnat_rules(network_settings, name),
+        snat_rules(network_settings, name),
+    ];
+    for rule_set in rule_sets {
+        clear_rules(&ipt, &rule_set)?;
+    }
+    Ok(())
+}
+
+#[cfg(target_os = "linux")]
+pub fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupError> {
+    for insert_rules in &rules.insert_rules {
+        clear_existing_chain_rules(ipt, insert_rules).map_err(|err| NetworkSetupError::Clean {
+            chain_name: insert_rules.chain_name.clone(),
+            message: err.to_string(),
+        })?;
+    }
+
+    for append_rules in &rules.append_rules {
+        clear_new_chain_rules(ipt, append_rules).map_err(|err| NetworkSetupError::Clean {
+            chain_name: append_rules.chain_name.clone(),
+            message: err.to_string(),
+        })?;
+    }
+
+    Ok(())
+}
+
+#[cfg(target_os = "linux")]
+pub fn clear_existing_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
+    let exists = ipt.chain_exists(rules.table_name, &rules.chain_name)?;
+    if !exists {
+        tracing::warn!("Can't clean the chain {}: doesn't exist", rules.chain_name);
+        return Ok(());
+    }
+
+    for rule in &rules.rules {
+        if ipt.exists(rules.table_name, &rules.chain_name, rule)? {
+            ipt.delete(rules.table_name, &rules.chain_name, rule)?;
+        } else {
+            tracing::warn!(
+                "Can't clean a rule for the chain {chain_name}: doesn't exist; rule: {rule}",
+                chain_name = rules.chain_name,
+            )
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(target_os = "linux")]
+pub fn clear_new_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
+    clear_existing_chain_rules(ipt, rules)?;
+    ipt.delete_chain(rules.table_name, &rules.chain_name)?;
+    Ok(())
+}
+
+#[cfg(target_os = "linux")]
+pub fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
+    for append_rules in &rules_set.append_rules {
+        ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
+        for r in &append_rules.rules {
+            ipt.append(append_rules.table_name, &append_rules.chain_name, r)?;
+        }
+    }
+
+    for rule in &rules_set.insert_rules {
+        for r in &rule.rules {
+            // 1 is a default position when inserting a rule
+            ipt.insert(rule.table_name, &rule.chain_name, r, 1)?;
+        }
+    }
+
+    Ok(())
+}
diff --git a/crates/vm-network-utils/src/non_linux_mocks.rs b/crates/vm-network-utils/src/non_linux_mocks.rs
new file mode 100644
index 0000000000..bb286196f2
--- /dev/null
+++ b/crates/vm-network-utils/src/non_linux_mocks.rs
@@ -0,0 +1,36 @@
+/*
+ * Nox Fluence Peer
+ *
+ * Copyright (C) 2024 Fluence DAO
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#[cfg(not(target_os = "linux"))]
+pub type IPTables = ();
+
+use crate::{IpTablesError, IpTablesRules, NetworkSettings, NetworkSetupError, RulesSet};
+
+#[cfg(not(target_os = "linux"))]
+pub fn clear_rules(_: &IPTables, _: &RulesSet) -> Result<(), NetworkSetupError> { Ok(()) }
+#[cfg(not(target_os = "linux"))]
+pub fn clear_network(_: &NetworkSettings, _: &str) -> Result<(), NetworkSetupError> { Ok(())}
+#[cfg(not(target_os = "linux"))]
+pub fn setup_network(_: &NetworkSettings, _: &str) -> Result<(), NetworkSetupError> { Ok(()) }
+#[cfg(not(target_os = "linux"))]
+pub fn clear_existing_chain_rules(_: &IPTables, _: &IpTablesRules) -> Result<(), IpTablesError> { Ok (()) }
+#[cfg(not(target_os = "linux"))]
+pub fn clear_new_chain_rules(_: &IPTables, _: &IpTablesRules) -> Result<(), IpTablesError> { Ok (()) }
+#[cfg(not(target_os = "linux"))]
+pub fn add_rules(_: &IPTables, _: &crate::RulesSet) -> Result<(), IpTablesError> { Ok(()) }

From 08072e481affd671968606050e31a50d60a5b63d Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 17:16:20 +0300
Subject: [PATCH 19/32] fix: add imports to linux.rs

---
 crates/vm-network-utils/src/linux.rs | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/crates/vm-network-utils/src/linux.rs b/crates/vm-network-utils/src/linux.rs
index d4cb8bb8f9..06ada48e8a 100644
--- a/crates/vm-network-utils/src/linux.rs
+++ b/crates/vm-network-utils/src/linux.rs
@@ -17,10 +17,11 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-#[cfg(target_os = "linux")]
+#![cfg(target_os = "linux")]
+
 use iptables::IPTables;
+use crate::{IpTablesError, IpTablesRules, NetworkSettings, NetworkSetupError, RulesSet};
 
-#[cfg(target_os = "linux")]
 pub fn setup_network(
     network_settings: &NetworkSettings,
     name: &str,
@@ -40,7 +41,6 @@ pub fn setup_network(
     Ok(())
 }
 
-#[cfg(target_os = "linux")]
 pub fn clear_network(
     network_settings: &NetworkSettings,
     name: &str,
@@ -59,7 +59,6 @@ pub fn clear_network(
     Ok(())
 }
 
-#[cfg(target_os = "linux")]
 pub fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupError> {
     for insert_rules in &rules.insert_rules {
         clear_existing_chain_rules(ipt, insert_rules).map_err(|err| NetworkSetupError::Clean {
@@ -78,7 +77,6 @@ pub fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupE
     Ok(())
 }
 
-#[cfg(target_os = "linux")]
 pub fn clear_existing_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
     let exists = ipt.chain_exists(rules.table_name, &rules.chain_name)?;
     if !exists {
@@ -100,14 +98,12 @@ pub fn clear_existing_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Resu
     Ok(())
 }
 
-#[cfg(target_os = "linux")]
 pub fn clear_new_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
     clear_existing_chain_rules(ipt, rules)?;
     ipt.delete_chain(rules.table_name, &rules.chain_name)?;
     Ok(())
 }
 
-#[cfg(target_os = "linux")]
 pub fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
     for append_rules in &rules_set.append_rules {
         ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;

From 66c4bf99b6e114781ac1f7c243592e086dd27dbd Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 17:30:08 +0300
Subject: [PATCH 20/32] fix: hope to fix compilation errors

---
 crates/vm-network-utils/src/linux.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/crates/vm-network-utils/src/linux.rs b/crates/vm-network-utils/src/linux.rs
index 06ada48e8a..ea8b42bead 100644
--- a/crates/vm-network-utils/src/linux.rs
+++ b/crates/vm-network-utils/src/linux.rs
@@ -19,8 +19,8 @@
 
 #![cfg(target_os = "linux")]
 
-use iptables::IPTables;
-use crate::{IpTablesError, IpTablesRules, NetworkSettings, NetworkSetupError, RulesSet};
+pub use iptables::IPTables;
+use crate::{IpTablesError, IpTablesRules, NetworkSettings, NetworkSetupError, RulesSet, setup_snat, setup_dnat, setup_fwd, fwd_rules, dnat_rules, snat_rules};
 
 pub fn setup_network(
     network_settings: &NetworkSettings,

From 3736b865339fc0bb4f690d5106c1fddc8a51a2e9 Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 17:37:58 +0300
Subject: [PATCH 21/32] fix: allow unused imports

---
 crates/vm-network-utils/src/linux.rs           | 2 ++
 crates/vm-network-utils/src/non_linux_mocks.rs | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/crates/vm-network-utils/src/linux.rs b/crates/vm-network-utils/src/linux.rs
index ea8b42bead..76c02f5a6a 100644
--- a/crates/vm-network-utils/src/linux.rs
+++ b/crates/vm-network-utils/src/linux.rs
@@ -17,6 +17,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+#![cfg_attr(not(target_os = "linux"), allow(unused_imports))]
+
 #![cfg(target_os = "linux")]
 
 pub use iptables::IPTables;
diff --git a/crates/vm-network-utils/src/non_linux_mocks.rs b/crates/vm-network-utils/src/non_linux_mocks.rs
index bb286196f2..bbcb144310 100644
--- a/crates/vm-network-utils/src/non_linux_mocks.rs
+++ b/crates/vm-network-utils/src/non_linux_mocks.rs
@@ -17,6 +17,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+#![cfg_attr(target_os = "linux", allow(unused_imports))]
+
 #[cfg(not(target_os = "linux"))]
 pub type IPTables = ();
 

From ab6ed30a17d403a9baf6137ed936ce58dba70377 Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 17:53:34 +0300
Subject: [PATCH 22/32] fix: export visibility levels

---
 crates/vm-network-utils/src/linux.rs          | 20 +++++++++----
 .../vm-network-utils/src/non_linux_mocks.rs   | 28 +++++++++++++++----
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/crates/vm-network-utils/src/linux.rs b/crates/vm-network-utils/src/linux.rs
index 76c02f5a6a..6b7ff783d7 100644
--- a/crates/vm-network-utils/src/linux.rs
+++ b/crates/vm-network-utils/src/linux.rs
@@ -18,11 +18,13 @@
  */
 
 #![cfg_attr(not(target_os = "linux"), allow(unused_imports))]
-
 #![cfg(target_os = "linux")]
 
+use crate::{
+    dnat_rules, fwd_rules, setup_dnat, setup_fwd, setup_snat, snat_rules, IpTablesError,
+    IpTablesRules, NetworkSettings, NetworkSetupError, RulesSet,
+};
 pub use iptables::IPTables;
-use crate::{IpTablesError, IpTablesRules, NetworkSettings, NetworkSetupError, RulesSet, setup_snat, setup_dnat, setup_fwd, fwd_rules, dnat_rules, snat_rules};
 
 pub fn setup_network(
     network_settings: &NetworkSettings,
@@ -61,7 +63,7 @@ pub fn clear_network(
     Ok(())
 }
 
-pub fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupError> {
+pub(crate) fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupError> {
     for insert_rules in &rules.insert_rules {
         clear_existing_chain_rules(ipt, insert_rules).map_err(|err| NetworkSetupError::Clean {
             chain_name: insert_rules.chain_name.clone(),
@@ -79,7 +81,10 @@ pub fn clear_rules(ipt: &IPTables, rules: &RulesSet) -> Result<(), NetworkSetupE
     Ok(())
 }
 
-pub fn clear_existing_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
+pub(crate) fn clear_existing_chain_rules(
+    ipt: &IPTables,
+    rules: &IpTablesRules,
+) -> Result<(), IpTablesError> {
     let exists = ipt.chain_exists(rules.table_name, &rules.chain_name)?;
     if !exists {
         tracing::warn!("Can't clean the chain {}: doesn't exist", rules.chain_name);
@@ -100,13 +105,16 @@ pub fn clear_existing_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Resu
     Ok(())
 }
 
-pub fn clear_new_chain_rules(ipt: &IPTables, rules: &IpTablesRules) -> Result<(), IpTablesError> {
+pub(crate) fn clear_new_chain_rules(
+    ipt: &IPTables,
+    rules: &IpTablesRules,
+) -> Result<(), IpTablesError> {
     clear_existing_chain_rules(ipt, rules)?;
     ipt.delete_chain(rules.table_name, &rules.chain_name)?;
     Ok(())
 }
 
-pub fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
+pub(crate) fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
     for append_rules in &rules_set.append_rules {
         ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
         for r in &append_rules.rules {
diff --git a/crates/vm-network-utils/src/non_linux_mocks.rs b/crates/vm-network-utils/src/non_linux_mocks.rs
index bbcb144310..ca518e6e3d 100644
--- a/crates/vm-network-utils/src/non_linux_mocks.rs
+++ b/crates/vm-network-utils/src/non_linux_mocks.rs
@@ -25,14 +25,30 @@ pub type IPTables = ();
 use crate::{IpTablesError, IpTablesRules, NetworkSettings, NetworkSetupError, RulesSet};
 
 #[cfg(not(target_os = "linux"))]
-pub fn clear_rules(_: &IPTables, _: &RulesSet) -> Result<(), NetworkSetupError> { Ok(()) }
+pub fn clear_network(_: &NetworkSettings, _: &str) -> Result<(), NetworkSetupError> {
+    Ok(())
+}
 #[cfg(not(target_os = "linux"))]
-pub fn clear_network(_: &NetworkSettings, _: &str) -> Result<(), NetworkSetupError> { Ok(())}
+pub fn setup_network(_: &NetworkSettings, _: &str) -> Result<(), NetworkSetupError> {
+    Ok(())
+}
+
 #[cfg(not(target_os = "linux"))]
-pub fn setup_network(_: &NetworkSettings, _: &str) -> Result<(), NetworkSetupError> { Ok(()) }
+pub(crate) fn clear_existing_chain_rules(
+    _: &IPTables,
+    _: &IpTablesRules,
+) -> Result<(), IpTablesError> {
+    Ok(())
+}
 #[cfg(not(target_os = "linux"))]
-pub fn clear_existing_chain_rules(_: &IPTables, _: &IpTablesRules) -> Result<(), IpTablesError> { Ok (()) }
+pub(crate) fn clear_new_chain_rules(_: &IPTables, _: &IpTablesRules) -> Result<(), IpTablesError> {
+    Ok(())
+}
 #[cfg(not(target_os = "linux"))]
-pub fn clear_new_chain_rules(_: &IPTables, _: &IpTablesRules) -> Result<(), IpTablesError> { Ok (()) }
+pub(crate) fn add_rules(_: &IPTables, _: &crate::RulesSet) -> Result<(), IpTablesError> {
+    Ok(())
+}
 #[cfg(not(target_os = "linux"))]
-pub fn add_rules(_: &IPTables, _: &crate::RulesSet) -> Result<(), IpTablesError> { Ok(()) }
+pub(crate) fn clear_rules(_: &IPTables, _: &RulesSet) -> Result<(), NetworkSetupError> {
+    Ok(())
+}

From 9a87b57da0fd7b041ab126138f02d1512f6a347e Mon Sep 17 00:00:00 2001
From: folex <0xdxdy@gmail.com>
Date: Wed, 28 Aug 2024 19:01:08 +0300
Subject: [PATCH 23/32] fix(vm_config): br0 => virbr0

---
 crates/server-config/src/node_config.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index f13b4e18ce..4726dc73c3 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -650,7 +650,7 @@ fn default_libvirt_uri() -> String {
 }
 
 fn default_bridge_name() -> String {
-    String::from("br0")
+    String::from("virbr0")
 }
 
 fn default_host_ssh_port() -> u16 {

From c27946052f503f10203cdf1062b1537bc242a2ac Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 29 Aug 2024 14:08:51 +0200
Subject: [PATCH 24/32] allocate memory based on physical cores

---
 crates/core-distributor/src/types.rs | 8 ++++++++
 crates/vm-utils/src/vm_utils.rs      | 5 ++++-
 crates/workers/src/workers.rs        | 8 +++++---
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/crates/core-distributor/src/types.rs b/crates/core-distributor/src/types.rs
index 94ee26759c..f745f6ddf6 100644
--- a/crates/core-distributor/src/types.rs
+++ b/crates/core-distributor/src/types.rs
@@ -107,6 +107,14 @@ impl Assignment {
             .collect::<Vec<_>>()
     }
 
+    pub fn physical_core_count(&self) -> usize {
+        self.cuid_cores
+            .values()
+            .map(|cores| cores.physical_core_id)
+            .collect::<BTreeSet<PhysicalCoreId>>()
+            .len()
+    }
+
     pub fn pin_current_thread_with(&self, thread_pinner: &dyn ThreadPinner) {
         thread_pinner.pin_current_thread_to_cpuset(&self.logical_core_ids());
     }
diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index c6c7670d52..6299d11279 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -17,6 +17,7 @@ pub struct CreateVMDomainParams {
     name: String,
     image: PathBuf,
     cpus: NonEmpty<LogicalCoreId>,
+    cores_num: usize,
     bridge_name: String,
     allow_gpu: bool,
 }
@@ -26,6 +27,7 @@ impl CreateVMDomainParams {
         name: String,
         image: PathBuf,
         cpus: NonEmpty<LogicalCoreId>,
+        cores_num: usize,
         bridge_name: String,
         allow_gpu: bool,
     ) -> Self {
@@ -33,6 +35,7 @@ impl CreateVMDomainParams {
             name,
             image,
             cpus,
+            cores_num,
             bridge_name,
             allow_gpu,
         }
@@ -346,7 +349,7 @@ fn prepare_xml(
         }
         mapping.push_str(format!("<vcpupin vcpu='{index}' cpuset='{logical_id}'/>").as_str());
     }
-    let memory_in_kb = params.cpus.len() * 4 * 1024 * 1024; // 4Gbs per core
+    let memory_in_kb = params.cores_num * 4 * 1024 * 1024; // 4Gbs per core
     let gpu_pci_configuration = gpu_pci_location
         .iter()
         .map(prepare_pci_config)
diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index cf5f9b05cc..367a066602 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -673,14 +673,16 @@ impl Workers {
         image: &Path,
         vm_config: &VmConfig,
     ) -> Result<String, WorkersError> {
-        let assignment = {
+        let (cores_count, assignment) = {
             let guard = self.assignments.read();
             let assignment = guard
                 .get(&worker_id)
                 .ok_or_else(|| WorkersError::WorkerNotFound(worker_id))?;
 
-            NonEmpty::from_vec(assignment.logical_core_ids())
-                .ok_or_else(|| WorkersError::WrongAssignment)?
+            let logical_cores = NonEmpty::from_vec(assignment.logical_core_ids())
+                .ok_or_else(|| WorkersError::WrongAssignment)?;
+            let physical_cores_count = assignment.physical_core_count();
+            (physical_cores_count, logical_cores)
         };
 
         let file_name = &image

From 36cfc012cc599bc98fb48e455461d3d3e8af1e46 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 29 Aug 2024 14:11:46 +0200
Subject: [PATCH 25/32] fix prev commit

---
 crates/workers/src/workers.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index 367a066602..ddbaede14f 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -710,6 +710,7 @@ impl Workers {
             vm_name.clone(),
             worker_image,
             assignment,
+            cores_count,
             vm_config.network.bridge_name.clone(),
             vm_config.allow_gpu,
         );

From 082aa5f0db98e331778f845b04c11fd5b82689dd Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 29 Aug 2024 16:44:29 +0200
Subject: [PATCH 26/32] fix tests

---
 crates/vm-utils/src/vm_utils.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index 6299d11279..93b0a95a2a 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -395,6 +395,7 @@ mod tests {
                 name: "test-id".to_string(),
                 image: "test-image".into(),
                 cpus: nonempty![1.into(), 8.into()],
+                cores_num: 2,
                 bridge_name: "br422442".to_string(),
                 allow_gpu: true,
             },
@@ -431,6 +432,7 @@ mod tests {
                 name: "test-id".to_string(),
                 image: "test-image".into(),
                 cpus: nonempty![1.into(), 8.into()],
+                cores_num: 2,
                 bridge_name: "br422442".to_string(),
                 allow_gpu: false,
             },
@@ -473,6 +475,7 @@ mod tests {
                 name: "test-id".to_string(),
                 image: image.clone(),
                 cpus: nonempty![1.into()],
+                cores_num: 1,
                 bridge_name: "br422442".to_string(),
                 allow_gpu: false,
             },

From dd481cd88d7b64156f12c734c32fc44b25aaf435 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 29 Aug 2024 16:45:52 +0200
Subject: [PATCH 27/32] fix fmt

---
 crates/server-config/src/node_config.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index 095f46d132..759921678d 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -653,7 +653,6 @@ fn default_vm_ip() -> Ipv4Addr {
     Ipv4Addr::new(192, 168, 122, 112)
 }
 
-
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct PortRangeConfig {
     pub start: u16,

From 3c6ec309c4a5349ed75282a8d9dfb87d2a78d9a9 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Thu, 29 Aug 2024 17:42:23 +0200
Subject: [PATCH 28/32] try to make create_vm restartable

---
 crates/vm-network-utils/src/linux.rs | 15 ++++++++++++++-
 crates/vm-utils/src/vm_utils.rs      | 12 +++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/crates/vm-network-utils/src/linux.rs b/crates/vm-network-utils/src/linux.rs
index 6b7ff783d7..3d587ac883 100644
--- a/crates/vm-network-utils/src/linux.rs
+++ b/crates/vm-network-utils/src/linux.rs
@@ -116,8 +116,17 @@ pub(crate) fn clear_new_chain_rules(
 
 pub(crate) fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
     for append_rules in &rules_set.append_rules {
-        ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
+        if ipt.chain_exists(append_rules.table_name, &append_rules.chain_name)? {
+            ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
+        } else {
+            tracing::info!("Chain {} already exists", append_rules.chain_name);
+        }
+
         for r in &append_rules.rules {
+            if ipt.exists(append_rules.table_name, &append_rules.chain_name, r)? {
+                tracing::info!("Rule already exists: {:?}", r);
+                continue;
+            }
             ipt.append(append_rules.table_name, &append_rules.chain_name, r)?;
         }
     }
@@ -125,6 +134,10 @@ pub(crate) fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTa
     for rule in &rules_set.insert_rules {
         for r in &rule.rules {
             // 1 is a default position when inserting a rule
+            if ipt.exists(rule.table_name, &rule.chain_name, r)? {
+                tracing::info!("Rule already exists: {:?}", r);
+                continue;
+            }
             ipt.insert(rule.table_name, &rule.chain_name, r, 1)?;
         }
     }
diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index 93b0a95a2a..79c777ae6d 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -214,11 +214,21 @@ pub fn start_vm(uri: &str, name: &str) -> Result<u32, VmError> {
         name: name.to_string(),
         err,
     })?;
-    domain.create().map_err(|err| VmError::FailedToStartVM {
+
+    let is_running = domain.is_active().map_err(|err| VmError::FailedToStartVM {
         err,
         name: name.to_string(),
     })?;
 
+    if is_running {
+        tracing::info!(target: "vm-utils","VM with name {name} is already running");
+    } else {
+        domain.create().map_err(|err| VmError::FailedToStartVM {
+            err,
+            name: name.to_string(),
+        })?;
+    }
+
     let id = domain.get_id().ok_or(VmError::FailedToGetVMId {
         name: name.to_string(),
     })?;

From 02d7f10969094c40b92dd391b104e83ec2cb5ff6 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Fri, 30 Aug 2024 12:56:59 +0200
Subject: [PATCH 29/32] fix the fix

---
 crates/vm-network-utils/src/linux.rs | 10 +++++++---
 crates/workers/src/workers.rs        |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/crates/vm-network-utils/src/linux.rs b/crates/vm-network-utils/src/linux.rs
index 3d587ac883..d6bed42569 100644
--- a/crates/vm-network-utils/src/linux.rs
+++ b/crates/vm-network-utils/src/linux.rs
@@ -110,16 +110,20 @@ pub(crate) fn clear_new_chain_rules(
     rules: &IpTablesRules,
 ) -> Result<(), IpTablesError> {
     clear_existing_chain_rules(ipt, rules)?;
-    ipt.delete_chain(rules.table_name, &rules.chain_name)?;
+    let exists = ipt.chain_exists(rules.table_name, &rules.chain_name)?;
+    if exists {
+        ipt.delete_chain(rules.table_name, &rules.chain_name)?;
+    }
+
     Ok(())
 }
 
 pub(crate) fn add_rules(ipt: &IPTables, rules_set: &RulesSet) -> Result<(), IpTablesError> {
     for append_rules in &rules_set.append_rules {
         if ipt.chain_exists(append_rules.table_name, &append_rules.chain_name)? {
-            ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
-        } else {
             tracing::info!("Chain {} already exists", append_rules.chain_name);
+        } else {
+            ipt.new_chain(append_rules.table_name, &append_rules.chain_name)?;
         }
 
         for r in &append_rules.rules {
diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs
index ddbaede14f..ea7605a28d 100644
--- a/crates/workers/src/workers.rs
+++ b/crates/workers/src/workers.rs
@@ -725,7 +725,7 @@ impl Workers {
         };
         if let Err(err) = result {
             // Clear the network on errors, so we can retry later (setup_network isn't idempotent)
-            tracing::warn!("couldn't create network or start VM, cleaning up network");
+            tracing::warn!("couldn't create network or start VM, cleaning up network: {err}");
             vm_network_utils::clear_network(&vm_config.network, vm_name.as_str())?;
             return Err(err);
         }

From 8757f10b1d86b664dd47ef0e494604373188356d Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Fri, 30 Aug 2024 15:25:51 +0200
Subject: [PATCH 30/32] fix defaults

---
 crates/server-config/src/node_config.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs
index 4726dc73c3..0aa87966ab 100644
--- a/crates/server-config/src/node_config.rs
+++ b/crates/server-config/src/node_config.rs
@@ -654,7 +654,7 @@ fn default_bridge_name() -> String {
 }
 
 fn default_host_ssh_port() -> u16 {
-    2222
+    922
 }
 
 fn default_vm_ssh_port() -> u16 {

From a34fb1a774082eceb72a29c693ba3894d0cb6016 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Fri, 30 Aug 2024 17:08:03 +0200
Subject: [PATCH 31/32] try to fix destroy

---
 crates/vm-utils/src/vm_utils.rs | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index 79c777ae6d..f8c81d38ff 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -113,7 +113,7 @@ pub enum VmError {
 }
 
 // The list of states is taken from the libvirt documentation
-#[derive(Debug)]
+#[derive(Debug, PartialEq, Eq)]
 pub enum VmStatus {
     NoState,
     Running,
@@ -191,12 +191,20 @@ pub fn remove_domain(uri: &str, name: &str) -> Result<(), VmError> {
         err,
     })?;
 
-    domain
-        .destroy()
-        .map_err(|err| VmError::FailedToRemoveVMDomain {
-            err,
+    if let Err(error) = domain.destroy() {
+        tracing::warn!(target: "vm-utils","Failed to destroy VM {name}: {error}");
+        let status = get_status(&domain).map_err(|err| VmError::FailedToGetInfo {
             name: name.to_string(),
+            err,
         })?;
+        if status != VmStatus::Shutoff {
+            tracing::warn!(target: "vm-utils","VM {name} is not in the shutoff state: {status}");
+            return Err(VmError::FailedToRemoveVMDomain {
+                err: error,
+                name: name.to_string(),
+            });
+        }
+    }
 
     domain
         .undefine()
@@ -288,10 +296,14 @@ pub fn status_vm(uri: &str, name: &str) -> Result<VmStatus, VmError> {
         name: name.to_string(),
         err,
     })?;
-    let info = domain.get_info().map_err(|err| VmError::FailedToGetInfo {
+    get_status(&domain).map_err(|err| VmError::FailedToGetInfo {
         name: name.to_string(),
         err,
-    })?;
+    })
+}
+
+fn get_status(domain: &Domain) -> Result<VmStatus, virt::error::Error> {
+    let info = domain.get_info()?;
 
     Ok(VmStatus::from_u32(info.state))
 }

From c11f38879b9ef769d4fba6772d34acad929bde55 Mon Sep 17 00:00:00 2001
From: Maria Kuklina <maria@fluence.one>
Date: Fri, 30 Aug 2024 17:29:09 +0200
Subject: [PATCH 32/32] fix pause

---
 crates/vm-utils/src/vm_utils.rs | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/crates/vm-utils/src/vm_utils.rs b/crates/vm-utils/src/vm_utils.rs
index f8c81d38ff..a71eb8a044 100644
--- a/crates/vm-utils/src/vm_utils.rs
+++ b/crates/vm-utils/src/vm_utils.rs
@@ -197,8 +197,8 @@ pub fn remove_domain(uri: &str, name: &str) -> Result<(), VmError> {
             name: name.to_string(),
             err,
         })?;
+        tracing::warn!(target: "vm-utils","VM {name} in the state: {status}");
         if status != VmStatus::Shutoff {
-            tracing::warn!(target: "vm-utils","VM {name} is not in the shutoff state: {status}");
             return Err(VmError::FailedToRemoveVMDomain {
                 err: error,
                 name: name.to_string(),
@@ -315,10 +315,21 @@ pub fn pause_vm(uri: &str, name: &str) -> Result<(), VmError> {
         name: name.to_string(),
         err,
     })?;
-    domain.suspend().map_err(|err| VmError::FailedToSuspendVM {
-        name: name.to_string(),
-        err,
-    })?;
+
+    if let Err(err) = domain.suspend() {
+        tracing::warn!(target: "vm-utils","Failed to suspend VM {name}: {err}");
+        let status = get_status(&domain).map_err(|err| VmError::FailedToGetInfo {
+            name: name.to_string(),
+            err,
+        })?;
+        tracing::warn!(target: "vm-utils","VM {name} in the state: {status}");
+        if status == VmStatus::Running {
+            return Err(VmError::FailedToSuspendVM {
+                name: name.to_string(),
+                err,
+            });
+        }
+    }
     Ok(())
 }