From 2a70820e7ac93a36062714736db7ce74b893a2b3 Mon Sep 17 00:00:00 2001 From: Vanient Date: Thu, 7 Sep 2023 14:29:42 +0800 Subject: [PATCH] vmm: support starting stratovirt lightweight vm sandbox vmm-sandboxer support starting stratovirt lightweight vm sandbox, which uses lightweight microvm mainboard and mmio bus. Signed-off-by: Vanient --- vmm/sandbox/src/stratovirt/config.rs | 33 +++++++++----- vmm/sandbox/src/stratovirt/devices/block.rs | 4 +- vmm/sandbox/src/stratovirt/devices/mod.rs | 6 +-- vmm/sandbox/src/stratovirt/devices/rng.rs | 4 +- vmm/sandbox/src/stratovirt/devices/serial.rs | 2 + .../src/stratovirt/devices/vhost_user_fs.rs | 2 + vmm/sandbox/src/stratovirt/devices/vsock.rs | 2 + vmm/sandbox/src/stratovirt/factory.rs | 43 +++++++++++-------- vmm/sandbox/src/stratovirt/mod.rs | 33 +++++++++----- 9 files changed, 83 insertions(+), 46 deletions(-) diff --git a/vmm/sandbox/src/stratovirt/config.rs b/vmm/sandbox/src/stratovirt/config.rs index c3f32c83..4a438b2a 100644 --- a/vmm/sandbox/src/stratovirt/config.rs +++ b/vmm/sandbox/src/stratovirt/config.rs @@ -20,8 +20,8 @@ use sandbox_derive::{CmdLineParamSet, CmdLineParams}; use serde::{Deserialize, Serialize}; use crate::{ - param::ToCmdLineParams, stratovirt::virtiofs::DEFAULT_VHOST_USER_FS_BIN_PATH, - vm::HypervisorCommonConfig, + device::Transport, param::ToCmdLineParams, + stratovirt::virtiofs::DEFAULT_VHOST_USER_FS_BIN_PATH, vm::HypervisorCommonConfig, }; #[allow(dead_code)] @@ -31,17 +31,15 @@ pub(crate) const MACHINE_TYPE_PC: &str = "pc"; #[allow(dead_code)] pub(crate) const MACHINE_TYPE_VIRT: &str = "virt"; #[allow(dead_code)] +pub(crate) const MACHINE_TYPE_MICROVM: &str = "microvm"; +#[allow(dead_code)] pub(crate) const MACHINE_TYPE_PSERIES: &str = "pseries"; #[allow(dead_code)] pub(crate) const MACHINE_TYPE_CCW_VIRTIO: &str = "s390-ccw-virtio"; const DEFAULT_STRATOVIRT_PATH: &str = "/usr/bin/stratovirt"; const DEFAULT_KERNEL_PARAMS: &str = "console=hvc0 console=hvc1 iommu=off panic=1 pcie_ports=native"; - -#[cfg(target_arch = "x86_64")] const ROOTFS_KERNEL_PARAMS: &str = " root=/dev/vda ro rootfstype=ext4"; -#[cfg(target_arch = "aarch64")] -const ROOTFS_KERNEL_PARAMS: &str = " root=/dev/vda1 ro rootfstype=ext4"; #[derive(Clone, Debug, Deserialize)] pub struct StratoVirtVMConfig { @@ -62,8 +60,8 @@ impl Default for StratoVirtVMConfig { fn default() -> Self { Self { common: Default::default(), - path: "stratovirt".to_string(), - machine_type: "virt".to_string(), + path: DEFAULT_STRATOVIRT_PATH.to_string(), + machine_type: MACHINE_TYPE_VIRT.to_string(), virtiofsd_conf: VirtiofsdConfig { path: DEFAULT_VHOST_USER_FS_BIN_PATH.to_string(), }, @@ -134,9 +132,12 @@ impl StratoVirtVMConfig { result.kernel.kernel_params.push_str(" debug task.debug"); } - result.global_params = vec![Global { - param: "pcie-root-port.fast-unplug=1".to_string(), - }]; + let machine_array: Vec<_> = self.machine_type.split(',').collect(); + if machine_array[0] != MACHINE_TYPE_MICROVM { + result.global_params = vec![Global { + param: "pcie-root-port.fast-unplug=1".to_string(), + }]; + } result.knobs = Knobs { daemonize: true, @@ -163,6 +164,16 @@ pub struct Machine { pub options: Option, } +impl Machine { + pub fn transport(&self) -> Transport { + let machine_array: Vec<_> = self.r#type.split(',').collect(); + match machine_array[0] { + MACHINE_TYPE_MICROVM => Transport::Mmio, + _ => Transport::Pci, + } + } +} + #[derive(CmdLineParamSet, Debug, Clone, Default, Serialize, Deserialize)] pub struct Kernel { #[param(key = "kernel")] diff --git a/vmm/sandbox/src/stratovirt/devices/block.rs b/vmm/sandbox/src/stratovirt/devices/block.rs index b6f7ebd4..5818049e 100644 --- a/vmm/sandbox/src/stratovirt/devices/block.rs +++ b/vmm/sandbox/src/stratovirt/devices/block.rs @@ -54,9 +54,9 @@ pub struct VirtioBlockDevice { pub readonly: Option, #[property(param = "drive", generator = "crate::utils::bool_to_on_off")] pub direct: Option, - #[property(param = "device")] + #[property(param = "device", predicate = "self.addr.len()>0")] pub bus: Option, - #[property(param = "device")] + #[property(param = "device", predicate = "self.addr.len()>0")] pub addr: String, } diff --git a/vmm/sandbox/src/stratovirt/devices/mod.rs b/vmm/sandbox/src/stratovirt/devices/mod.rs index 353eee45..41f57167 100644 --- a/vmm/sandbox/src/stratovirt/devices/mod.rs +++ b/vmm/sandbox/src/stratovirt/devices/mod.rs @@ -72,7 +72,7 @@ pub trait StratoVirtHotAttachable: Device + HotAttachable {} impl StratoVirtHotAttachable for T where T: Device + HotAttachable {} -pub fn create_pcie_root_bus() -> PcieRootBus { +pub fn create_pcie_root_bus() -> Option { let mut pcie_root_bus = PcieRootBus { id: "pcie.0".to_string(), bus: Bus { @@ -85,7 +85,7 @@ pub fn create_pcie_root_bus() -> PcieRootBus { // since pcie.0/0x0 addr is reserved, set slot 0 status is "SlotStatus::Occupied" pcie_root_bus.bus.slots[0].status = SlotStatus::Occupied("reserved".to_string()); - pcie_root_bus + Some(pcie_root_bus) } #[cfg(test)] @@ -96,7 +96,7 @@ mod tests { #[test] fn test_create_pcie_root_bus() { let pcie_root_bus = create_pcie_root_bus(); - if let SlotStatus::Occupied(s) = &pcie_root_bus.bus.slots[0].status { + if let SlotStatus::Occupied(s) = &pcie_root_bus.unwrap().bus.slots[0].status { assert_eq!(s, "reserved"); } } diff --git a/vmm/sandbox/src/stratovirt/devices/rng.rs b/vmm/sandbox/src/stratovirt/devices/rng.rs index 78cf7774..3f95ce9e 100644 --- a/vmm/sandbox/src/stratovirt/devices/rng.rs +++ b/vmm/sandbox/src/stratovirt/devices/rng.rs @@ -38,9 +38,9 @@ pub struct VirtioRngDevice { pub(crate) max_bytes: Option, #[property(param = "device")] pub(crate) period: Option, - #[property(param = "device")] + #[property(param = "device", predicate = "self.addr.len()>0")] pub(crate) bus: String, - #[property(param = "device")] + #[property(param = "device", predicate = "self.addr.len()>0")] pub(crate) addr: String, } diff --git a/vmm/sandbox/src/stratovirt/devices/serial.rs b/vmm/sandbox/src/stratovirt/devices/serial.rs index 92ea0e4d..b117b0f6 100644 --- a/vmm/sandbox/src/stratovirt/devices/serial.rs +++ b/vmm/sandbox/src/stratovirt/devices/serial.rs @@ -26,7 +26,9 @@ pub struct SerialDevice { #[property(ignore_key)] pub driver: String, pub id: String, + #[property(param = "device", predicate = "self.addr.len()>0")] pub bus: String, + #[property(param = "device", predicate = "self.addr.len()>0")] pub addr: String, #[property(ignore)] pub transport: Transport, diff --git a/vmm/sandbox/src/stratovirt/devices/vhost_user_fs.rs b/vmm/sandbox/src/stratovirt/devices/vhost_user_fs.rs index 4e275fad..89e012c4 100644 --- a/vmm/sandbox/src/stratovirt/devices/vhost_user_fs.rs +++ b/vmm/sandbox/src/stratovirt/devices/vhost_user_fs.rs @@ -30,7 +30,9 @@ pub struct VhostUserFs { #[property(key = "chardev")] pub chardev_id: String, pub tag: String, + #[property(param = "device", predicate = "self.addr.len()>0")] pub bus: String, + #[property(param = "device", predicate = "self.addr.len()>0")] pub addr: String, } diff --git a/vmm/sandbox/src/stratovirt/devices/vsock.rs b/vmm/sandbox/src/stratovirt/devices/vsock.rs index 31d0f266..2239f011 100644 --- a/vmm/sandbox/src/stratovirt/devices/vsock.rs +++ b/vmm/sandbox/src/stratovirt/devices/vsock.rs @@ -39,7 +39,9 @@ pub struct VSockDevice { pub id: String, #[property(key = "guest-cid")] pub context_id: u64, + #[property(param = "device", predicate = "self.addr.len()>0")] pub bus: String, + #[property(param = "device", predicate = "self.addr.len()>0")] pub addr: String, pub vhostfd: i32, } diff --git a/vmm/sandbox/src/stratovirt/factory.rs b/vmm/sandbox/src/stratovirt/factory.rs index 48b24f8d..5307b25d 100644 --- a/vmm/sandbox/src/stratovirt/factory.rs +++ b/vmm/sandbox/src/stratovirt/factory.rs @@ -32,9 +32,8 @@ use super::devices::{ DEFAULT_SERIAL_DEVICE_ID, PCIE_ROOTPORT_CAPACITY, }; use crate::{ - device::Transport, stratovirt::{ - config::{QmpSocket, StratoVirtVMConfig}, + config::{QmpSocket, StratoVirtVMConfig, MACHINE_TYPE_MICROVM}, devices::vsock::{find_context_id, VSockDevice}, StratoVirtVM, }, @@ -82,21 +81,30 @@ impl VMFactory for StratoVirtVMFactory { no_wait: true, }); - // create pcie.0 root bus object - vm.pcie_root_bus = create_pcie_root_bus(); + let machine_clone = vm.config.machine.r#type.clone(); + let machine_array: Vec<_> = machine_clone.split(',').collect(); + if machine_array[0] != MACHINE_TYPE_MICROVM { + // create pcie.0 root bus object + vm.pcie_root_bus = create_pcie_root_bus(); + } + let transport = vm.config.machine.transport(); // set virtio-rng device let rng_device = VirtioRngDevice::new( DEFAULT_RNG_DEVICE_ID, "/dev/urandom", - Transport::Pci, + transport.clone(), DEFAULT_PCIE_BUS, ); - vm.attach_to_pcie_rootbus(rng_device)?; + vm.attach_to_bus(rng_device)?; // set console - let serial = SerialDevice::new(DEFAULT_SERIAL_DEVICE_ID, Transport::Pci, DEFAULT_PCIE_BUS); - vm.attach_to_pcie_rootbus(serial)?; + let serial = SerialDevice::new( + DEFAULT_SERIAL_DEVICE_ID, + transport.clone(), + DEFAULT_PCIE_BUS, + ); + vm.attach_to_bus(serial)?; let console_backend_chardev = CharDevice::new("socket", DEFAULT_CONSOLE_CHARDEV_ID, &vm.console_socket); vm.attach_device(console_backend_chardev); @@ -106,22 +114,22 @@ impl VMFactory for StratoVirtVMFactory { if vm.config.kernel.image.is_some() { let mut image_device: VirtioBlockDevice = VirtioBlockDevice::new( - &Transport::Pci.to_driver(VIRTIO_BLK_DRIVER), + &transport.clone().to_driver(VIRTIO_BLK_DRIVER), "rootfs", "blk-0", vm.config.kernel.image.clone(), Some(true), ); image_device.bus = Some(DEFAULT_PCIE_BUS.to_string()); - vm.attach_to_pcie_rootbus(image_device)?; + vm.attach_to_bus(image_device)?; } // set vsock port as the rpc channel to agent let (fd, cid) = find_context_id().await?; let fd_index = vm.append_fd(fd); let vhost_vsock_device = - VSockDevice::new(cid, Transport::Pci, DEFAULT_PCIE_BUS, fd_index as i32); - vm.attach_to_pcie_rootbus(vhost_vsock_device)?; + VSockDevice::new(cid, transport.clone(), DEFAULT_PCIE_BUS, fd_index as i32); + vm.attach_to_bus(vhost_vsock_device)?; vm.agent_socket = format!("vsock://{}:1024", cid); //share fs, stratovirt only support virtiofs share @@ -133,12 +141,12 @@ impl VMFactory for StratoVirtVMFactory { vm.attach_device(virtiofs_chardev); let vhost_user_fs_device = VhostUserFs::new( &format!("vhost-user-fs-{}", id), - Transport::Pci, + transport.clone(), &chardev_id, DEFAULT_MOUNT_TAG_NAME, DEFAULT_PCIE_BUS, ); - vm.attach_to_pcie_rootbus(vhost_user_fs_device)?; + vm.attach_to_bus(vhost_user_fs_device)?; // create virtiofs daemon vm.create_vitiofs_daemon( @@ -146,9 +154,10 @@ impl VMFactory for StratoVirtVMFactory { s.base_dir.as_str(), share_fs_path.as_str(), ); - - // set pcie-root-ports for hotplugging - vm.create_pcie_root_ports(PCIE_ROOTPORT_CAPACITY)?; + if machine_array[0] != MACHINE_TYPE_MICROVM { + // set pcie-root-ports for hotplugging + vm.create_pcie_root_ports(PCIE_ROOTPORT_CAPACITY)?; + } Ok(vm) } diff --git a/vmm/sandbox/src/stratovirt/mod.rs b/vmm/sandbox/src/stratovirt/mod.rs index 30bb2f29..5f7a5801 100644 --- a/vmm/sandbox/src/stratovirt/mod.rs +++ b/vmm/sandbox/src/stratovirt/mod.rs @@ -92,7 +92,7 @@ pub struct StratoVirtVM { client: Option, virtiofs_daemon: Option, #[serde(skip)] - pcie_root_bus: PcieRootBus, + pcie_root_bus: Option, #[serde(skip)] pcie_root_ports_pool: Option, } @@ -199,7 +199,7 @@ impl VM for StratoVirtVM { .vhostfds(vec![]) .bus(Some(DEFAULT_PCIE_BUS.to_string())) .build(); - self.attach_to_pcie_rootbus(virtio_net_device)?; + self.attach_to_bus(virtio_net_device)?; } _ => { todo!() @@ -292,7 +292,7 @@ impl StratoVirtVM { client: None, virtiofs_daemon: None, pcie_root_ports_pool: None, - pcie_root_bus: PcieRootBus::default(), + pcie_root_bus: None, pids: Pids::default(), } } @@ -435,7 +435,15 @@ impl StratoVirtVM { } fn get_empty_pcie_slot_index(&mut self) -> Result { - for (index, slot) in self.pcie_root_bus.bus.slots.iter_mut().enumerate() { + for (index, slot) in self + .pcie_root_bus + .as_mut() + .unwrap() + .bus + .slots + .iter_mut() + .enumerate() + { if let SlotStatus::Empty = slot.status { return Ok(index); } @@ -446,15 +454,18 @@ impl StratoVirtVM { )) } - fn attach_to_pcie_rootbus( + fn attach_to_bus( &mut self, mut device: T, ) -> Result<()> { - // get the empty slot index - let slot_index = self.get_empty_pcie_slot_index()?; - // set the pcie slot status to Occupied - self.pcie_root_bus.bus.slots[slot_index].status = SlotStatus::Occupied(device.id()); - device.set_device_addr(slot_index); + if self.pcie_root_bus.is_some() { + // get the empty slot index + let slot_index = self.get_empty_pcie_slot_index()?; + // set the pcie slot status to Occupied + self.pcie_root_bus.as_mut().unwrap().bus.slots[slot_index].status = + SlotStatus::Occupied(device.id()); + device.set_device_addr(slot_index); + } self.devices.push(Box::new(device)); Ok(()) } @@ -480,7 +491,7 @@ impl StratoVirtVM { None, ); root_ports.push(root_port.clone()); - self.attach_to_pcie_rootbus(root_port)?; + self.attach_to_bus(root_port)?; } self.pcie_root_ports_pool = Some(PCIERootPorts {