From 0bb130afa680597f8d9188cf496feb406bd13bbd Mon Sep 17 00:00:00 2001 From: James Curtis Date: Wed, 28 Aug 2024 08:38:35 +0000 Subject: [PATCH] Implement hot-unplug on the Guest side Guest side hot-unplug implementation. This means that the API call can be made, and the vCPUs are successfully removed from the guest, but the backing vCPU threads are not removed from the host VMM. As a result of this, hot-plugs that occur after hot-unplugs do not work correctly right now. Once the total thread count in the VMM exceeds 32, there is no effect of hot-plugging. To complete this, a refactor is needed of the VMM, such that the CpuContainer can somehow remove the threads when the guest kernel calls _EJ0. Signed-off-by: James Curtis --- src/vmm/src/devices/acpi/cpu_container.rs | 52 ++++++++++++++++++++++- src/vmm/src/lib.rs | 48 ++++++++++++++++++++- src/vmm/src/rpc_interface.rs | 32 +++++++++++--- 3 files changed, 124 insertions(+), 8 deletions(-) diff --git a/src/vmm/src/devices/acpi/cpu_container.rs b/src/vmm/src/devices/acpi/cpu_container.rs index 54c86639d55..429ceb9d073 100644 --- a/src/vmm/src/devices/acpi/cpu_container.rs +++ b/src/vmm/src/devices/acpi/cpu_container.rs @@ -51,6 +51,8 @@ pub const CPU_CONTAINER_ACPI_SIZE: usize = 0xC; const CPU_ENABLE_BIT: u8 = 1 << 0; const CPU_INSERTING_BIT: u8 = 1 << 1; +const CPU_REMOVING_BIT: u8 = 1 << 2; +const CPU_EJECT_BIT: u8 = 1 << 3; const CPU_SELECTION_OFFSET: u64 = 0; const CPU_STATUS_OFFSET: u64 = 4; @@ -94,6 +96,7 @@ impl CpuContainer { cpu_id: i, active: i < boot_count, inserting: false, + removing: false, }) } @@ -123,6 +126,9 @@ impl CpuContainer { if cpu_device.inserting { data[0] |= CPU_INSERTING_BIT; } + if cpu_device.removing { + data[0] |= CPU_REMOVING_BIT; + } } else { error!("Out of range vCPU id: {}", self.selected_cpu) } @@ -143,6 +149,10 @@ impl CpuContainer { if data[0] & CPU_ENABLE_BIT != 0 { cpu_device.active = true; } + if data[0] & CPU_EJECT_BIT != 0 { + cpu_device.active = false; + // TODO: Remove vCPU handle from VMM + } } else { error!("Out of range vCPU id: {}", self.selected_cpu) } @@ -215,7 +225,9 @@ impl Aml for CpuContainer { aml::FieldEntry::Reserved(32), aml::FieldEntry::Named(*b"CPEN", 1), aml::FieldEntry::Named(*b"CINS", 1), - aml::FieldEntry::Reserved(6), + aml::FieldEntry::Named(*b"CRMV", 1), + aml::FieldEntry::Named(*b"CEJ0", 1), + aml::FieldEntry::Reserved(4), aml::FieldEntry::Named(*b"CCMD", 8), ], ), @@ -270,6 +282,8 @@ pub struct CpuDevice { pub active: bool, /// Whether this CPU is in the process of being inserted pub inserting: bool, + /// Whether this CPU is in the process of being removed + pub removing: bool, } impl CpuDevice { @@ -305,6 +319,16 @@ impl Aml for CpuDevice { ))], ), &aml::Name::new("_MAT".into(), &aml::Buffer::new(mat_data)), + &aml::Method::new( + "_EJ0".into(), + 1, + false, + // Call into CEJ0 method which will actually eject device + vec![&aml::MethodCall::new( + "\\_SB_.CPUS.CEJ0".into(), + vec![&self.cpu_id], + )], + ), ], ) .append_aml_bytes(v) @@ -320,7 +344,7 @@ impl Aml for CpuNotify { let object = aml::Path::new(&format!("C{:03X}", self.cpu_id)); aml::If::new( &aml::Equal::new(&aml::Arg(0), &self.cpu_id), - vec![&aml::Notify::new(&object, &1u8)], + vec![&aml::Notify::new(&object, &aml::Arg(1))], ) .append_aml_bytes(v) } @@ -369,6 +393,21 @@ impl Aml for CpuMethods { aml::Method::new("CTFY".into(), 2, true, cpu_notifies_refs).append_aml_bytes(v); + aml::Method::new( + "CEJ0".into(), + 1, + true, + vec![ + &aml::Acquire::new("\\_SB_.PRES.CPLK".into(), 0xffff), + // Write CPU number (in first argument) to I/O port via field + &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CSEL"), &aml::Arg(0)), + // Set CEJ0 bit + &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CEJ0"), &aml::ONE), + &aml::Release::new("\\_SB_.PRES.CPLK".into()), + ], + ) + .append_aml_bytes(v); + aml::Method::new( "CSCN".into(), 0, @@ -396,6 +435,15 @@ impl Aml for CpuMethods { &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CINS"), &aml::ONE), ], ), + &aml::If::new( + &aml::Equal::new(&aml::Path::new("\\_SB_.PRES.CRMV"), &aml::ONE), + // Notify device if it is (with the eject constant 0x3) + vec![ + &aml::MethodCall::new("CTFY".into(), vec![&aml::Local(0), &3u8]), + // Reset CRMV bit + &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CRMV"), &aml::ONE), + ], + ), &aml::Add::new(&aml::Local(0), &aml::Local(0), &aml::ONE), ], ), diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 0ed613527af..d11fa27f849 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -615,7 +615,7 @@ impl Vmm { config: HotplugVcpuConfig, ) -> Result { use crate::logger::IncMetric; - if config.target > MAX_SUPPORTED_VCPUS.into() { + if config.target > MAX_SUPPORTED_VCPUS { return Err(HotplugVcpuError::VcpuCountTooHigh); } @@ -682,6 +682,52 @@ impl Vmm { } /// Removes vCPUs from VMM. + #[cfg(target_arch = "x86_64")] + pub fn hotunplug_vcpus( + &mut self, + config: HotplugVcpuConfig, + ) -> Result { + use crate::logger::IncMetric; + if config.target < 1 { + return Err(HotplugVcpuError::VcpuCountTooLow); + } + if let Some(kvm_config) = self.vcpu_config.as_mut() { + kvm_config.vcpu_count = config.target; + } + + #[allow(clippy::cast_possible_truncation)] + let start_idx: u8 = config.target; + if let Some(devices::BusDevice::CpuContainer(cont)) = + self.get_bus_device(DeviceType::CpuContainer, "CpuContainer") + { + let mut locked_container = cont.lock().expect("Poisoned lock"); + for cpu_idx in start_idx..u8::try_from(self.vcpus_handles.len()).unwrap() { + locked_container.cpu_devices[cpu_idx as usize].removing = true; + } + } + + #[allow(clippy::cast_lossless)] + METRICS + .hotplug + .vcpus_added + .add(self.vcpus_handles.len() as u64 - config.target as u64); + + // Update VM config to reflect new CPUs added + #[allow(clippy::cast_possible_truncation)] + let new_machine_config = MachineConfigUpdate { + vcpu_count: Some(self.vcpus_handles.len() as u8), + mem_size_mib: None, + smt: None, + cpu_template: None, + track_dirty_pages: None, + huge_pages: None, + }; + + self.acpi_device_manager.notify_cpu_container()?; + + Ok(new_machine_config) + } + /// Retrieves the KVM dirty bitmap for each of the guest's memory regions. pub fn reset_dirty_bitmap(&self) { self.guest_memory diff --git a/src/vmm/src/rpc_interface.rs b/src/vmm/src/rpc_interface.rs index ae09a18f571..ae55c5e4567 100644 --- a/src/vmm/src/rpc_interface.rs +++ b/src/vmm/src/rpc_interface.rs @@ -670,7 +670,17 @@ impl RuntimeApiController { self.vmm.lock().expect("Poisoned lock").version(), )), #[cfg(target_arch = "x86_64")] - HotplugRequest(request_type) => self.handle_hotplug_request(request_type), + HotplugRequest(request_type) => { + let curr_vcpus: u8 = self + .vmm + .lock() + .expect("Poisoned lock") + .vcpus_handles + .len() + .try_into() + .unwrap(); + self.handle_hotplug_request(request_type, curr_vcpus) + } PatchMMDS(value) => self.patch_mmds(value), Pause => self.pause(), PutMMDS(value) => self.put_mmds(value), @@ -872,13 +882,25 @@ impl RuntimeApiController { fn handle_hotplug_request( &mut self, cfg: HotplugRequestConfig, + curr_vcpus: u8, ) -> Result { match cfg { HotplugRequestConfig::Vcpu(cfg) => { - let result = self.vmm.lock().expect("Poisoned lock").hotplug_vcpus(cfg); - result - .map_err(|err| VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err))) - .and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update)) + if cfg.target > curr_vcpus { + let result = self.vmm.lock().expect("Poisoned lock").hotplug_vcpus(cfg); + result + .map_err(|err| { + VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err)) + }) + .and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update)) + } else { + let result = self.vmm.lock().expect("Poisoned lock").hotunplug_vcpus(cfg); + result + .map_err(|err| { + VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err)) + }) + .and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update)) + } } } }