Skip to content

Commit

Permalink
Implement hot-unplug on the Guest side
Browse files Browse the repository at this point in the history 
Guest side hot-unplug implementation. This means that the API call can
be made, and the vCPUs are successfully removed from the guest, but the
backing vCPU threads are not removed from the host VMM.

As a result of this, hot-plugs that occur after hot-unplugs do not work
correctly right now. Once the total thread count in the VMM exceeds 32,
there is no effect of hot-plugging. To complete this, a refactor is
needed of the VMM, such that the CpuContainer can somehow remove the
threads when the guest kernel calls _EJ0.

Signed-off-by: James Curtis <[email protected]>
  • Loading branch information
@JamesC1305
JamesC1305 committed Aug 28, 2024
1 parent 2a87db8 commit 0bb130a
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 8 deletions.
52 changes: 50 additions & 2 deletions src/vmm/src/devices/acpi/cpu_container.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ pub const CPU_CONTAINER_ACPI_SIZE: usize = 0xC;

const CPU_ENABLE_BIT: u8 = 1 << 0;
const CPU_INSERTING_BIT: u8 = 1 << 1;
const CPU_REMOVING_BIT: u8 = 1 << 2;
const CPU_EJECT_BIT: u8 = 1 << 3;

const CPU_SELECTION_OFFSET: u64 = 0;
const CPU_STATUS_OFFSET: u64 = 4;
Expand Down Expand Up @@ -94,6 +96,7 @@ impl CpuContainer {
cpu_id: i,
active: i < boot_count,
inserting: false,
removing: false,
})
}

Expand Down Expand Up @@ -123,6 +126,9 @@ impl CpuContainer {
if cpu_device.inserting {
data[0] |= CPU_INSERTING_BIT;
}
if cpu_device.removing {
data[0] |= CPU_REMOVING_BIT;
}
} else {
error!("Out of range vCPU id: {}", self.selected_cpu)
}
Expand All @@ -143,6 +149,10 @@ impl CpuContainer {
if data[0] & CPU_ENABLE_BIT != 0 {
cpu_device.active = true;
}
if data[0] & CPU_EJECT_BIT != 0 {
cpu_device.active = false;
// TODO: Remove vCPU handle from VMM
}
} else {
error!("Out of range vCPU id: {}", self.selected_cpu)
}
Expand Down Expand Up @@ -215,7 +225,9 @@ impl Aml for CpuContainer {
aml::FieldEntry::Reserved(32),
aml::FieldEntry::Named(*b"CPEN", 1),
aml::FieldEntry::Named(*b"CINS", 1),
aml::FieldEntry::Reserved(6),
aml::FieldEntry::Named(*b"CRMV", 1),
aml::FieldEntry::Named(*b"CEJ0", 1),
aml::FieldEntry::Reserved(4),
aml::FieldEntry::Named(*b"CCMD", 8),
],
),
Expand Down Expand Up @@ -270,6 +282,8 @@ pub struct CpuDevice {
pub active: bool,
/// Whether this CPU is in the process of being inserted
pub inserting: bool,
/// Whether this CPU is in the process of being removed
pub removing: bool,
}

impl CpuDevice {
Expand Down Expand Up @@ -305,6 +319,16 @@ impl Aml for CpuDevice {
))],
),
&aml::Name::new("_MAT".into(), &aml::Buffer::new(mat_data)),
&aml::Method::new(
"_EJ0".into(),
1,
false,
// Call into CEJ0 method which will actually eject device
vec![&aml::MethodCall::new(
"\\_SB_.CPUS.CEJ0".into(),
vec![&self.cpu_id],
)],
),
],
)
.append_aml_bytes(v)
Expand All @@ -320,7 +344,7 @@ impl Aml for CpuNotify {
let object = aml::Path::new(&format!("C{:03X}", self.cpu_id));
aml::If::new(
&aml::Equal::new(&aml::Arg(0), &self.cpu_id),
vec![&aml::Notify::new(&object, &1u8)],
vec![&aml::Notify::new(&object, &aml::Arg(1))],
)
.append_aml_bytes(v)
}
Expand Down Expand Up @@ -369,6 +393,21 @@ impl Aml for CpuMethods {

aml::Method::new("CTFY".into(), 2, true, cpu_notifies_refs).append_aml_bytes(v);

aml::Method::new(
"CEJ0".into(),
1,
true,
vec![
&aml::Acquire::new("\\_SB_.PRES.CPLK".into(), 0xffff),
// Write CPU number (in first argument) to I/O port via field
&aml::Store::new(&aml::Path::new("\\_SB_.PRES.CSEL"), &aml::Arg(0)),
// Set CEJ0 bit
&aml::Store::new(&aml::Path::new("\\_SB_.PRES.CEJ0"), &aml::ONE),
&aml::Release::new("\\_SB_.PRES.CPLK".into()),
],
)
.append_aml_bytes(v);

aml::Method::new(
"CSCN".into(),
0,
Expand Down Expand Up @@ -396,6 +435,15 @@ impl Aml for CpuMethods {
&aml::Store::new(&aml::Path::new("\\_SB_.PRES.CINS"), &aml::ONE),
],
),
&aml::If::new(
&aml::Equal::new(&aml::Path::new("\\_SB_.PRES.CRMV"), &aml::ONE),
// Notify device if it is (with the eject constant 0x3)
vec![
&aml::MethodCall::new("CTFY".into(), vec![&aml::Local(0), &3u8]),
// Reset CRMV bit
&aml::Store::new(&aml::Path::new("\\_SB_.PRES.CRMV"), &aml::ONE),
],
),
&aml::Add::new(&aml::Local(0), &aml::Local(0), &aml::ONE),
],
),
Expand Down
48 changes: 47 additions & 1 deletion src/vmm/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,7 +615,7 @@ impl Vmm {
config: HotplugVcpuConfig,
) -> Result<MachineConfigUpdate, HotplugVcpuError> {
use crate::logger::IncMetric;
if config.target > MAX_SUPPORTED_VCPUS.into() {
if config.target > MAX_SUPPORTED_VCPUS {
return Err(HotplugVcpuError::VcpuCountTooHigh);
}

Expand Down Expand Up @@ -682,6 +682,52 @@ impl Vmm {
}

/// Removes vCPUs from VMM.
#[cfg(target_arch = "x86_64")]
pub fn hotunplug_vcpus(
&mut self,
config: HotplugVcpuConfig,
) -> Result<MachineConfigUpdate, HotplugVcpuError> {
use crate::logger::IncMetric;
if config.target < 1 {
return Err(HotplugVcpuError::VcpuCountTooLow);
}
if let Some(kvm_config) = self.vcpu_config.as_mut() {
kvm_config.vcpu_count = config.target;
}

#[allow(clippy::cast_possible_truncation)]
let start_idx: u8 = config.target;
if let Some(devices::BusDevice::CpuContainer(cont)) =
self.get_bus_device(DeviceType::CpuContainer, "CpuContainer")
{
let mut locked_container = cont.lock().expect("Poisoned lock");
for cpu_idx in start_idx..u8::try_from(self.vcpus_handles.len()).unwrap() {
locked_container.cpu_devices[cpu_idx as usize].removing = true;
}
}

#[allow(clippy::cast_lossless)]
METRICS
.hotplug
.vcpus_added
.add(self.vcpus_handles.len() as u64 - config.target as u64);

// Update VM config to reflect new CPUs added
#[allow(clippy::cast_possible_truncation)]
let new_machine_config = MachineConfigUpdate {
vcpu_count: Some(self.vcpus_handles.len() as u8),
mem_size_mib: None,
smt: None,
cpu_template: None,
track_dirty_pages: None,
huge_pages: None,
};

self.acpi_device_manager.notify_cpu_container()?;

Ok(new_machine_config)
}

/// Retrieves the KVM dirty bitmap for each of the guest's memory regions.
pub fn reset_dirty_bitmap(&self) {
self.guest_memory
Expand Down
32 changes: 27 additions & 5 deletions src/vmm/src/rpc_interface.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -670,7 +670,17 @@ impl RuntimeApiController {
self.vmm.lock().expect("Poisoned lock").version(),
)),
#[cfg(target_arch = "x86_64")]
HotplugRequest(request_type) => self.handle_hotplug_request(request_type),
HotplugRequest(request_type) => {
let curr_vcpus: u8 = self
.vmm
.lock()
.expect("Poisoned lock")
.vcpus_handles
.len()
.try_into()
.unwrap();
self.handle_hotplug_request(request_type, curr_vcpus)
}
PatchMMDS(value) => self.patch_mmds(value),
Pause => self.pause(),
PutMMDS(value) => self.put_mmds(value),
Expand Down Expand Up @@ -872,13 +882,25 @@ impl RuntimeApiController {
fn handle_hotplug_request(
&mut self,
cfg: HotplugRequestConfig,
curr_vcpus: u8,
) -> Result<VmmData, VmmActionError> {
match cfg {
HotplugRequestConfig::Vcpu(cfg) => {
let result = self.vmm.lock().expect("Poisoned lock").hotplug_vcpus(cfg);
result
.map_err(|err| VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err)))
.and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update))
if cfg.target > curr_vcpus {
let result = self.vmm.lock().expect("Poisoned lock").hotplug_vcpus(cfg);
result
.map_err(|err| {
VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err))
})
.and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update))
} else {
let result = self.vmm.lock().expect("Poisoned lock").hotunplug_vcpus(cfg);
result
.map_err(|err| {
VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err))
})
.and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update))
}
}
}
}
Expand Down

0 comments on commit 0bb130a

Please sign in to comment.