vmm: support starting stratovirt lightweight vm sandbox

vmm-sandboxer support starting stratovirt lightweight vm sandbox,
which uses lightweight microvm mainboard and mmio bus.

Signed-off-by: Vanient <[email protected]>

vmm/sandbox/src/stratovirt/config.rs

@@ -20,8 +20,8 @@ use sandbox_derive::{CmdLineParamSet, CmdLineParams};
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    param::ToCmdLineParams, stratovirt::virtiofs::DEFAULT_VHOST_USER_FS_BIN_PATH,
-    vm::HypervisorCommonConfig,
+    device::Transport, param::ToCmdLineParams,
+    stratovirt::virtiofs::DEFAULT_VHOST_USER_FS_BIN_PATH, vm::HypervisorCommonConfig,
 };
 
 #[allow(dead_code)]
@@ -31,17 +31,15 @@ pub(crate) const MACHINE_TYPE_PC: &str = "pc";
 #[allow(dead_code)]
 pub(crate) const MACHINE_TYPE_VIRT: &str = "virt";
 #[allow(dead_code)]
+pub(crate) const MACHINE_TYPE_MICROVM: &str = "microvm";
+#[allow(dead_code)]
 pub(crate) const MACHINE_TYPE_PSERIES: &str = "pseries";
 #[allow(dead_code)]
 pub(crate) const MACHINE_TYPE_CCW_VIRTIO: &str = "s390-ccw-virtio";
 
 const DEFAULT_STRATOVIRT_PATH: &str = "/usr/bin/stratovirt";
 const DEFAULT_KERNEL_PARAMS: &str = "console=hvc0 console=hvc1 iommu=off panic=1 pcie_ports=native";
-
-#[cfg(target_arch = "x86_64")]
 const ROOTFS_KERNEL_PARAMS: &str = " root=/dev/vda ro rootfstype=ext4";
-#[cfg(target_arch = "aarch64")]
-const ROOTFS_KERNEL_PARAMS: &str = " root=/dev/vda1 ro rootfstype=ext4";
 
 #[derive(Clone, Debug, Deserialize)]
 pub struct StratoVirtVMConfig {
@@ -62,8 +60,8 @@ impl Default for StratoVirtVMConfig {
     fn default() -> Self {
         Self {
             common: Default::default(),
-            path: "stratovirt".to_string(),
-            machine_type: "virt".to_string(),
+            path: DEFAULT_STRATOVIRT_PATH.to_string(),
+            machine_type: MACHINE_TYPE_VIRT.to_string(),
             virtiofsd_conf: VirtiofsdConfig {
                 path: DEFAULT_VHOST_USER_FS_BIN_PATH.to_string(),
             },
@@ -134,9 +132,12 @@ impl StratoVirtVMConfig {
             result.kernel.kernel_params.push_str(" debug task.debug");
         }
 
-        result.global_params = vec![Global {
-            param: "pcie-root-port.fast-unplug=1".to_string(),
-        }];
+        let machine_array: Vec<_> = self.machine_type.split(',').collect();
+        if machine_array[0] != MACHINE_TYPE_MICROVM {
+            result.global_params = vec![Global {
+                param: "pcie-root-port.fast-unplug=1".to_string(),
+            }];
+        }
 
         result.knobs = Knobs {
             daemonize: true,
@@ -163,6 +164,16 @@ pub struct Machine {
     pub options: Option<String>,
 }
 
+impl Machine {
+    pub fn transport(&self) -> Transport {
+        let machine_array: Vec<_> = self.r#type.split(',').collect();
+        match machine_array[0] {
+            MACHINE_TYPE_MICROVM => Transport::Mmio,
+            _ => Transport::Pci,
+        }
+    }
+}
+
 #[derive(CmdLineParamSet, Debug, Clone, Default, Serialize, Deserialize)]
 pub struct Kernel {
     #[param(key = "kernel")]

vmm/sandbox/src/stratovirt/devices/block.rs

@@ -54,9 +54,9 @@ pub struct VirtioBlockDevice {
     pub readonly: Option<bool>,
     #[property(param = "drive", generator = "crate::utils::bool_to_on_off")]
     pub direct: Option<bool>,
-    #[property(param = "device")]
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub bus: Option<String>,
-    #[property(param = "device")]
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub addr: String,
 }
 

vmm/sandbox/src/stratovirt/devices/mod.rs

@@ -72,7 +72,7 @@ pub trait StratoVirtHotAttachable: Device + HotAttachable {}
 
 impl<T> StratoVirtHotAttachable for T where T: Device + HotAttachable {}
 
-pub fn create_pcie_root_bus() -> PcieRootBus {
+pub fn create_pcie_root_bus() -> Option<PcieRootBus> {
     let mut pcie_root_bus = PcieRootBus {
         id: "pcie.0".to_string(),
         bus: Bus {
@@ -85,7 +85,7 @@ pub fn create_pcie_root_bus() -> PcieRootBus {
 
     // since pcie.0/0x0 addr is reserved, set slot 0 status is "SlotStatus::Occupied"
     pcie_root_bus.bus.slots[0].status = SlotStatus::Occupied("reserved".to_string());
-    pcie_root_bus
+    Some(pcie_root_bus)
 }
 
 #[cfg(test)]
@@ -96,7 +96,7 @@ mod tests {
     #[test]
     fn test_create_pcie_root_bus() {
         let pcie_root_bus = create_pcie_root_bus();
-        if let SlotStatus::Occupied(s) = &pcie_root_bus.bus.slots[0].status {
+        if let SlotStatus::Occupied(s) = &pcie_root_bus.unwrap().bus.slots[0].status {
             assert_eq!(s, "reserved");
         }
     }

vmm/sandbox/src/stratovirt/devices/rng.rs

@@ -38,9 +38,9 @@ pub struct VirtioRngDevice {
     pub(crate) max_bytes: Option<u32>,
     #[property(param = "device")]
     pub(crate) period: Option<u32>,
-    #[property(param = "device")]
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub(crate) bus: String,
-    #[property(param = "device")]
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub(crate) addr: String,
 }
 

vmm/sandbox/src/stratovirt/devices/serial.rs

@@ -26,7 +26,9 @@ pub struct SerialDevice {
     #[property(ignore_key)]
     pub driver: String,
     pub id: String,
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub bus: String,
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub addr: String,
     #[property(ignore)]
     pub transport: Transport,

vmm/sandbox/src/stratovirt/devices/vhost_user_fs.rs

@@ -30,7 +30,9 @@ pub struct VhostUserFs {
     #[property(key = "chardev")]
     pub chardev_id: String,
     pub tag: String,
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub bus: String,
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub addr: String,
 }
 

vmm/sandbox/src/stratovirt/devices/vsock.rs

@@ -39,7 +39,9 @@ pub struct VSockDevice {
     pub id: String,
     #[property(key = "guest-cid")]
     pub context_id: u64,
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub bus: String,
+    #[property(param = "device", predicate = "self.addr.len()>0")]
     pub addr: String,
     pub vhostfd: i32,
 }

vmm/sandbox/src/stratovirt/factory.rs

@@ -32,9 +32,8 @@ use super::devices::{
     DEFAULT_SERIAL_DEVICE_ID, PCIE_ROOTPORT_CAPACITY,
 };
 use crate::{
-    device::Transport,
     stratovirt::{
-        config::{QmpSocket, StratoVirtVMConfig},
+        config::{QmpSocket, StratoVirtVMConfig, MACHINE_TYPE_MICROVM},
         devices::vsock::{find_context_id, VSockDevice},
         StratoVirtVM,
     },
@@ -82,21 +81,30 @@ impl VMFactory for StratoVirtVMFactory {
             no_wait: true,
         });
 
-        // create pcie.0 root bus object
-        vm.pcie_root_bus = create_pcie_root_bus();
+        let machine_clone = vm.config.machine.r#type.clone();
+        let machine_array: Vec<_> = machine_clone.split(',').collect();
+        if machine_array[0] != MACHINE_TYPE_MICROVM {
+            // create pcie.0 root bus object
+            vm.pcie_root_bus = create_pcie_root_bus();
+        }
 
+        let transport = vm.config.machine.transport();
         // set virtio-rng device
         let rng_device = VirtioRngDevice::new(
             DEFAULT_RNG_DEVICE_ID,
             "/dev/urandom",
-            Transport::Pci,
+            transport.clone(),
             DEFAULT_PCIE_BUS,
         );
-        vm.attach_to_pcie_rootbus(rng_device)?;
+        vm.attach_to_bus(rng_device)?;
 
         // set console
-        let serial = SerialDevice::new(DEFAULT_SERIAL_DEVICE_ID, Transport::Pci, DEFAULT_PCIE_BUS);
-        vm.attach_to_pcie_rootbus(serial)?;
+        let serial = SerialDevice::new(
+            DEFAULT_SERIAL_DEVICE_ID,
+            transport.clone(),
+            DEFAULT_PCIE_BUS,
+        );
+        vm.attach_to_bus(serial)?;
         let console_backend_chardev =
             CharDevice::new("socket", DEFAULT_CONSOLE_CHARDEV_ID, &vm.console_socket);
         vm.attach_device(console_backend_chardev);
@@ -106,22 +114,22 @@ impl VMFactory for StratoVirtVMFactory {
 
         if vm.config.kernel.image.is_some() {
             let mut image_device: VirtioBlockDevice = VirtioBlockDevice::new(
-                &Transport::Pci.to_driver(VIRTIO_BLK_DRIVER),
+                &transport.clone().to_driver(VIRTIO_BLK_DRIVER),
                 "rootfs",
                 "blk-0",
                 vm.config.kernel.image.clone(),
                 Some(true),
             );
             image_device.bus = Some(DEFAULT_PCIE_BUS.to_string());
-            vm.attach_to_pcie_rootbus(image_device)?;
+            vm.attach_to_bus(image_device)?;
         }
 
         // set vsock port as the rpc channel to agent
         let (fd, cid) = find_context_id().await?;
         let fd_index = vm.append_fd(fd);
         let vhost_vsock_device =
-            VSockDevice::new(cid, Transport::Pci, DEFAULT_PCIE_BUS, fd_index as i32);
-        vm.attach_to_pcie_rootbus(vhost_vsock_device)?;
+            VSockDevice::new(cid, transport.clone(), DEFAULT_PCIE_BUS, fd_index as i32);
+        vm.attach_to_bus(vhost_vsock_device)?;
         vm.agent_socket = format!("vsock://{}:1024", cid);
 
         //share fs, stratovirt only support virtiofs share
@@ -133,22 +141,23 @@ impl VMFactory for StratoVirtVMFactory {
         vm.attach_device(virtiofs_chardev);
         let vhost_user_fs_device = VhostUserFs::new(
             &format!("vhost-user-fs-{}", id),
-            Transport::Pci,
+            transport.clone(),
             &chardev_id,
             DEFAULT_MOUNT_TAG_NAME,
             DEFAULT_PCIE_BUS,
         );
-        vm.attach_to_pcie_rootbus(vhost_user_fs_device)?;
+        vm.attach_to_bus(vhost_user_fs_device)?;
 
         // create virtiofs daemon
         vm.create_vitiofs_daemon(
             self.default_config.virtiofsd_conf.path.as_str(),
             s.base_dir.as_str(),
             share_fs_path.as_str(),
         );
-
-        // set pcie-root-ports for hotplugging
-        vm.create_pcie_root_ports(PCIE_ROOTPORT_CAPACITY)?;
+        if machine_array[0] != MACHINE_TYPE_MICROVM {
+            // set pcie-root-ports for hotplugging
+            vm.create_pcie_root_ports(PCIE_ROOTPORT_CAPACITY)?;
+        }
 
         Ok(vm)
     }

vmm/sandbox/src/stratovirt/mod.rs

@@ -92,7 +92,7 @@ pub struct StratoVirtVM {
     client: Option<QmpClient>,
     virtiofs_daemon: Option<VirtiofsDaemon>,
     #[serde(skip)]
-    pcie_root_bus: PcieRootBus,
+    pcie_root_bus: Option<PcieRootBus>,
     #[serde(skip)]
     pcie_root_ports_pool: Option<PCIERootPorts>,
 }
@@ -199,7 +199,7 @@ impl VM for StratoVirtVM {
                     .vhostfds(vec![])
                     .bus(Some(DEFAULT_PCIE_BUS.to_string()))
                     .build();
-                self.attach_to_pcie_rootbus(virtio_net_device)?;
+                self.attach_to_bus(virtio_net_device)?;
             }
             _ => {
                 todo!()
@@ -292,7 +292,7 @@ impl StratoVirtVM {
             client: None,
             virtiofs_daemon: None,
             pcie_root_ports_pool: None,
-            pcie_root_bus: PcieRootBus::default(),
+            pcie_root_bus: None,
             pids: Pids::default(),
         }
     }
@@ -435,7 +435,15 @@ impl StratoVirtVM {
     }
 
     fn get_empty_pcie_slot_index(&mut self) -> Result<usize> {
-        for (index, slot) in self.pcie_root_bus.bus.slots.iter_mut().enumerate() {
+        for (index, slot) in self
+            .pcie_root_bus
+            .as_mut()
+            .unwrap()
+            .bus
+            .slots
+            .iter_mut()
+            .enumerate()
+        {
             if let SlotStatus::Empty = slot.status {
                 return Ok(index);
             }
@@ -446,15 +454,18 @@ impl StratoVirtVM {
         ))
     }
 
-    fn attach_to_pcie_rootbus<T: StratoVirtDevice + Sync + Send + 'static>(
+    fn attach_to_bus<T: StratoVirtDevice + Sync + Send + 'static>(
         &mut self,
         mut device: T,
     ) -> Result<()> {
-        // get the empty slot index
-        let slot_index = self.get_empty_pcie_slot_index()?;
-        // set the pcie slot status to Occupied
-        self.pcie_root_bus.bus.slots[slot_index].status = SlotStatus::Occupied(device.id());
-        device.set_device_addr(slot_index);
+        if self.pcie_root_bus.is_some() {
+            // get the empty slot index
+            let slot_index = self.get_empty_pcie_slot_index()?;
+            // set the pcie slot status to Occupied
+            self.pcie_root_bus.as_mut().unwrap().bus.slots[slot_index].status =
+                SlotStatus::Occupied(device.id());
+            device.set_device_addr(slot_index);
+        }
         self.devices.push(Box::new(device));
         Ok(())
     }
@@ -480,7 +491,7 @@ impl StratoVirtVM {
                 None,
             );
             root_ports.push(root_port.clone());
-            self.attach_to_pcie_rootbus(root_port)?;
+            self.attach_to_bus(root_port)?;
         }
 
         self.pcie_root_ports_pool = Some(PCIERootPorts {