From a541725ad39856bc956f58bc930aef35f3a0eaa6 Mon Sep 17 00:00:00 2001 From: James Curtis Date: Thu, 11 Jul 2024 14:20:51 +0000 Subject: [PATCH] Configure KVM vCPUS and resume new threads after creation Properly configure KVM vCPUs so that the configuration is normalized and matches that of already existing KVM vCPUs. Signed-off-by: James Curtis --- src/vmm/src/builder.rs | 7 +++ src/vmm/src/lib.rs | 41 +++++++++++++--- src/vmm/src/rpc_interface.rs | 18 +++++++ src/vmm/src/vmm_config/hotplug.rs | 8 ++- src/vmm/src/vstate/vcpu/mod.rs | 2 +- src/vmm/src/vstate/vcpu/x86_64.rs | 82 ++++++++++++++++++++++--------- 6 files changed, 126 insertions(+), 32 deletions(-) diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index 4f5c1e64d3a..66e5b29ec3d 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -236,6 +236,8 @@ fn create_vmm_and_vcpus( vcpus_handles: Vec::new(), vcpus_exit_evt, #[cfg(target_arch = "x86_64")] + vcpu_config: None, + #[cfg(target_arch = "x86_64")] seccomp_filters, resource_allocator, mmio_device_manager, @@ -858,6 +860,9 @@ pub fn configure_system_for_boot( cpu_config, }; + #[cfg(target_arch = "x86_64")] + vmm.attach_vcpu_config(vcpu_config.clone()); + // Configure vCPUs with normalizing and setting the generated CPU configuration. for vcpu in vcpus.iter_mut() { vcpu.kvm_vcpu @@ -1259,6 +1264,8 @@ pub mod tests { vcpus_handles: Vec::new(), vcpus_exit_evt, #[cfg(target_arch = "x86_64")] + vcpu_config: None, + #[cfg(target_arch = "x86_64")] seccomp_filters: crate::seccomp_filters::get_empty_filters(), resource_allocator, mmio_device_manager, diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 30cc1ef7854..6d6445686a2 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -320,6 +320,9 @@ pub struct Vmm { vcpus_handles: Vec, // Used by Vcpus and devices to initiate teardown; Vmm should never write here. vcpus_exit_evt: EventFd, + // Used to configure kvm vcpus during hotplugging. + #[cfg(target_arch = "x86_64")] + vcpu_config: Option, // seccomp_filters are only needed in VMM for hotplugging vCPUS. #[cfg(target_arch = "x86_64")] seccomp_filters: BpfThreadMap, @@ -411,15 +414,25 @@ impl Vmm { pub fn resume_vm(&mut self) -> Result<(), VmmError> { self.mmio_device_manager.kick_devices(); - // Send the events. - self.vcpus_handles + self.resume_vcpu_threads(0)?; + + self.instance_info.state = VmState::Running; + Ok(()) + } + + /// Resume vCPU threads + fn resume_vcpu_threads(&mut self, start_idx: usize) -> Result<(), VmmError> { + if start_idx >= self.vcpus_handles.len() { + return Err(VmmError::VcpuMessage); + } + + self.vcpus_handles[start_idx..] .iter() .try_for_each(|handle| handle.send_event(VcpuEvent::Resume)) .map_err(|_| VmmError::VcpuMessage)?; // Check the responses. - if self - .vcpus_handles + if self.vcpus_handles[start_idx..] .iter() .map(|handle| handle.response_receiver().recv_timeout(RECV_TIMEOUT_SEC)) .any(|response| !matches!(response, Ok(VcpuResponse::Resumed))) @@ -427,7 +440,6 @@ impl Vmm { return Err(VmmError::VcpuMessage); } - self.instance_info.state = VmState::Running; Ok(()) } @@ -615,6 +627,9 @@ impl Vmm { return Err(HotplugVcpuError::VcpuCountTooHigh); } + if let Some(kvm_config) = self.vcpu_config.as_mut() { + kvm_config.vcpu_count += config.add; + } // Create and start new vcpus let mut vcpus = Vec::with_capacity(config.add.into()); @@ -629,8 +644,13 @@ impl Vmm { .vcpus_exit_evt .try_clone() .map_err(HotplugVcpuError::EventFd)?; - let vcpu = + let mut vcpu = Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?; + if let Some(kvm_config) = self.vcpu_config.as_ref() { + vcpu.kvm_vcpu.hotplug_configure(kvm_config)?; + } else { + return Err(HotplugVcpuError::RestoredFromSnapshot); + } locked_container.cpu_devices[cpu_idx as usize].inserting = true; vcpus.push(vcpu); } @@ -659,6 +679,8 @@ impl Vmm { huge_pages: None, }; + self.resume_vcpu_threads(start_idx.into())?; + self.acpi_device_manager.notify_cpu_container()?; Ok(new_machine_config) @@ -871,6 +893,13 @@ impl Vmm { } } + /// Add the vcpu configuration used during boot to the VMM. This is required as part of the + /// hotplugging process, to correctly configure KVM vCPUs. + #[cfg(target_arch = "x86_64")] + pub fn attach_vcpu_config(&mut self, vcpu_config: VcpuConfig) { + self.vcpu_config = Some(vcpu_config) + } + /// Signals Vmm to stop and exit. pub fn stop(&mut self, exit_code: FcExitCode) { // To avoid cycles, all teardown paths take the following route: diff --git a/src/vmm/src/rpc_interface.rs b/src/vmm/src/rpc_interface.rs index 6fa83ddfcd0..92db9a61e18 100644 --- a/src/vmm/src/rpc_interface.rs +++ b/src/vmm/src/rpc_interface.rs @@ -2149,8 +2149,23 @@ mod tests { #[test] #[cfg(target_arch = "x86_64")] fn test_runtime_hotplug_vcpu() { + use std::collections::BTreeMap; + + use crate::cpu_config::x86_64::cpuid::Cpuid; + // Case 1. Valid input let mut vmm = default_vmm(); + let cpuid = Cpuid::try_from(vmm.vm.supported_cpuid().clone()).unwrap(); + let vcpu_config = crate::VcpuConfig { + vcpu_count: 0, + smt: false, + cpu_config: crate::cpu_config::templates::CpuConfiguration { + cpuid, + msrs: BTreeMap::new(), + }, + }; + + vmm.attach_vcpu_config(vcpu_config.clone()); let config = HotplugVcpuConfig { add: 4 }; let result = vmm.hotplug_vcpus(config); assert_eq!(vmm.vcpus_handles.len(), 4); @@ -2158,6 +2173,7 @@ mod tests { // Case 2. Vcpu count too low let mut vmm = default_vmm(); + vmm.attach_vcpu_config(vcpu_config.clone()); vmm.hotplug_vcpus(HotplugVcpuConfig { add: 1 }).unwrap(); assert_eq!(vmm.vcpus_handles.len(), 1); let config = HotplugVcpuConfig { add: 0 }; @@ -2167,6 +2183,7 @@ mod tests { // Case 3. Vcpu count too high let mut vmm = default_vmm(); + vmm.attach_vcpu_config(vcpu_config.clone()); vmm.hotplug_vcpus(HotplugVcpuConfig { add: 1 }).unwrap(); assert_eq!(vmm.vcpus_handles.len(), 1); let config = HotplugVcpuConfig { add: 33 }; @@ -2176,6 +2193,7 @@ mod tests { // Case 4. Attempted overflow of vcpus let mut vmm = default_vmm(); + vmm.attach_vcpu_config(vcpu_config.clone()); vmm.hotplug_vcpus(HotplugVcpuConfig { add: 2 }).unwrap(); assert_eq!(vmm.vcpus_handles.len(), 2); let config = HotplugVcpuConfig { add: 255 }; diff --git a/src/vmm/src/vmm_config/hotplug.rs b/src/vmm/src/vmm_config/hotplug.rs index 66b2b72b138..d497dda1a0e 100644 --- a/src/vmm/src/vmm_config/hotplug.rs +++ b/src/vmm/src/vmm_config/hotplug.rs @@ -5,8 +5,8 @@ use std::io; use serde::{Deserialize, Serialize}; -use crate::vstate::vcpu::VcpuError; -use crate::StartVcpusError; +use crate::vstate::vcpu::{KvmVcpuConfigureError, VcpuError}; +use crate::{StartVcpusError, VmmError}; /// Unifying enum for all types of hotplug request configs. /// Currently only Vcpus may be hotplugged. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -34,10 +34,14 @@ pub enum HotplugVcpuError { EventFd(#[from] io::Error), /// Error creating the vcpu: {0} VcpuCreate(VcpuError), + /// Error configuring KVM vcpu: {0} + VcpuConfigure(#[from] KvmVcpuConfigureError), /// Failed to start vCPUs VcpuStart(StartVcpusError), /// No seccomp filter for thread category: {0} MissingSeccompFilters(String), + /// Error resuming VM: {0} + VmResume(#[from] VmmError), /// Cannot hotplug vCPUs after restoring from snapshot RestoredFromSnapshot, } diff --git a/src/vmm/src/vstate/vcpu/mod.rs b/src/vmm/src/vstate/vcpu/mod.rs index a8dad7feb21..1fa5bec09c2 100644 --- a/src/vmm/src/vstate/vcpu/mod.rs +++ b/src/vmm/src/vstate/vcpu/mod.rs @@ -63,7 +63,7 @@ pub enum VcpuError { } /// Encapsulates configuration parameters for the guest vCPUS. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct VcpuConfig { /// Number of guest VCPUs. pub vcpu_count: u8, diff --git a/src/vmm/src/vstate/vcpu/x86_64.rs b/src/vmm/src/vstate/vcpu/x86_64.rs index d91bfd1cea2..362fcbbb075 100644 --- a/src/vmm/src/vstate/vcpu/x86_64.rs +++ b/src/vmm/src/vstate/vcpu/x86_64.rs @@ -9,8 +9,8 @@ use std::collections::BTreeMap; use std::fmt::Debug; use kvm_bindings::{ - kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs, - kvm_xsave, CpuId, Msrs, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES, + kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_msr_entry, kvm_regs, kvm_sregs, + kvm_vcpu_events, kvm_xcrs, kvm_xsave, CpuId, Msrs, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES, }; use kvm_ioctls::{VcpuExit, VcpuFd}; use log::{error, warn}; @@ -178,22 +178,17 @@ impl KvmVcpu { }) } - /// Configures a x86_64 specific vcpu for booting Linux and should be called once per vcpu. - /// + /// General configuration - common for both boot and hotplugged CPUs. + /// Normalizes and sets the CPUID in KVM and creates KVM MSRs /// # Arguments - /// - /// * `guest_mem` - The guest memory used by this microvm. - /// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts. - /// * `vcpu_config` - The vCPU configuration. - /// * `cpuid` - The capabilities exposed by this vCPU. - pub fn configure( + /// * vcpu_config - The configuration for the vCPUs. + /// * msrs - The MSRs currently present. + fn configure_common( &mut self, - guest_mem: &GuestMemoryMmap, - kernel_start_addr: GuestAddress, vcpu_config: &VcpuConfig, - ) -> Result<(), KvmVcpuConfigureError> { + msrs: std::collections::BTreeMap, + ) -> Result<(Vec, CpuId), KvmVcpuConfigureError> { let mut cpuid = vcpu_config.cpu_config.cpuid.clone(); - // Apply machine specific changes to CPUID. cpuid.normalize( // The index of the current logical CPU in the range [0..cpu_count]. @@ -212,6 +207,35 @@ impl KvmVcpu { .set_cpuid2(&kvm_cpuid) .map_err(KvmVcpuConfigureError::SetCpuid)?; + // // Clone MSR entries that are modified by CPU template from `VcpuConfig`. + + let kvm_msrs = msrs + .clone() + .into_iter() + .map(|entry| kvm_bindings::kvm_msr_entry { + index: entry.0, + data: entry.1, + ..Default::default() + }) + .collect::>(); + + Ok((kvm_msrs, kvm_cpuid)) + } + + /// Configures a x86_64 specific vcpu for booting Linux and should be called once per vcpu. + /// + /// # Arguments + /// + /// * `guest_mem` - The guest memory used by this microvm. + /// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts. + /// * `vcpu_config` - The vCPU configuration. + /// * `cpuid` - The capabilities exposed by this vCPU. + pub fn configure( + &mut self, + guest_mem: &GuestMemoryMmap, + kernel_start_addr: GuestAddress, + vcpu_config: &VcpuConfig, + ) -> Result<(), KvmVcpuConfigureError> { // Clone MSR entries that are modified by CPU template from `VcpuConfig`. let mut msrs = vcpu_config.cpu_config.msrs.clone(); self.msrs_to_save.extend(msrs.keys()); @@ -221,6 +245,8 @@ impl KvmVcpu { msrs.insert(entry.index, entry.data); }); + let (kvm_msrs, kvm_cpuid) = self.configure_common(vcpu_config, msrs)?; + // TODO - Add/amend MSRs for vCPUs based on cpu_config // By this point the Guest CPUID is established. Some CPU features require MSRs // to configure and interact with those features. If a MSR is writable from @@ -239,15 +265,6 @@ impl KvmVcpu { // save is `architectural MSRs` + `MSRs inferred through CPUID` + `other // MSRs defined by the template` - let kvm_msrs = msrs - .into_iter() - .map(|entry| kvm_bindings::kvm_msr_entry { - index: entry.0, - data: entry.1, - ..Default::default() - }) - .collect::>(); - crate::arch::x86_64::msr::set_msrs(&self.fd, &kvm_msrs)?; crate::arch::x86_64::regs::setup_regs(&self.fd, kernel_start_addr.raw_value())?; crate::arch::x86_64::regs::setup_fpu(&self.fd)?; @@ -257,6 +274,25 @@ impl KvmVcpu { Ok(()) } + /// Configures an x86_64 cpu that has been hotplugged post-boot. Called once per hotplugged + /// vcpu. + /// + /// # Arguments + /// + /// * `vcpu_config` - The config to be applied to the vCPU, defined at boot time. + pub fn hotplug_configure( + &mut self, + vcpu_config: &VcpuConfig, + ) -> Result<(), KvmVcpuConfigureError> { + let msrs = vcpu_config.cpu_config.msrs.clone(); + let (kvm_msrs, _) = self.configure_common(vcpu_config, msrs)?; + + crate::arch::x86_64::msr::set_msrs(&self.fd, &kvm_msrs)?; + crate::arch::x86_64::interrupts::set_lint(&self.fd)?; + + Ok(()) + } + /// Sets a Port Mapped IO bus for this vcpu. pub fn set_pio_bus(&mut self, pio_bus: crate::devices::Bus) { self.peripherals.pio_bus = Some(pio_bus);