Configure KVM vCPUS and resume new threads after creation

Properly configure KVM vCPUs so that the configuration is normalized and
matches that of already existing KVM vCPUs.

Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>

Author: JamesC1305

src/vmm/src/builder.rs

@@ -236,6 +236,8 @@ fn create_vmm_and_vcpus(
         vcpus_handles: Vec::new(),
         vcpus_exit_evt,
         #[cfg(target_arch = "x86_64")]
+        vcpu_config: None,
+        #[cfg(target_arch = "x86_64")]
         seccomp_filters,
         resource_allocator,
         mmio_device_manager,
@@ -858,6 +860,9 @@ pub fn configure_system_for_boot(
         cpu_config,
     };
 
+    #[cfg(target_arch = "x86_64")]
+    vmm.attach_vcpu_config(vcpu_config.clone());
+
     // Configure vCPUs with normalizing and setting the generated CPU configuration.
     for vcpu in vcpus.iter_mut() {
         vcpu.kvm_vcpu
@@ -1259,6 +1264,8 @@ pub mod tests {
             vcpus_handles: Vec::new(),
             vcpus_exit_evt,
             #[cfg(target_arch = "x86_64")]
+            vcpu_config: None,
+            #[cfg(target_arch = "x86_64")]
             seccomp_filters: crate::seccomp_filters::get_empty_filters(),
             resource_allocator,
             mmio_device_manager,

src/vmm/src/lib.rs

@@ -320,6 +320,9 @@ pub struct Vmm {
     vcpus_handles: Vec<VcpuHandle>,
     // Used by Vcpus and devices to initiate teardown; Vmm should never write here.
     vcpus_exit_evt: EventFd,
+    // Used to configure kvm vcpus during hotplugging.
+    #[cfg(target_arch = "x86_64")]
+    vcpu_config: Option<VcpuConfig>,
     // seccomp_filters are only needed in VMM for hotplugging vCPUS.
     #[cfg(target_arch = "x86_64")]
     seccomp_filters: BpfThreadMap,
@@ -411,23 +414,32 @@ impl Vmm {
     pub fn resume_vm(&mut self) -> Result<(), VmmError> {
         self.mmio_device_manager.kick_devices();
 
-        // Send the events.
-        self.vcpus_handles
+        self.resume_vcpu_threads(0)?;
+
+        self.instance_info.state = VmState::Running;
+        Ok(())
+    }
+
+    /// Resume vCPU threads
+    fn resume_vcpu_threads(&mut self, start_idx: usize) -> Result<(), VmmError> {
+        if start_idx >= self.vcpus_handles.len() {
+            return Err(VmmError::VcpuMessage);
+        }
+
+        self.vcpus_handles[start_idx..]
             .iter()
             .try_for_each(|handle| handle.send_event(VcpuEvent::Resume))
             .map_err(|_| VmmError::VcpuMessage)?;
 
         // Check the responses.
-        if self
-            .vcpus_handles
+        if self.vcpus_handles[start_idx..]
             .iter()
             .map(|handle| handle.response_receiver().recv_timeout(RECV_TIMEOUT_SEC))
             .any(|response| !matches!(response, Ok(VcpuResponse::Resumed)))
         {
             return Err(VmmError::VcpuMessage);
         }
 
-        self.instance_info.state = VmState::Running;
         Ok(())
     }
 
@@ -615,6 +627,9 @@ impl Vmm {
             return Err(HotplugVcpuError::VcpuCountTooHigh);
         }
 
+        if let Some(kvm_config) = self.vcpu_config.as_mut() {
+            kvm_config.vcpu_count += config.add;
+        }
         // Create and start new vcpus
         let mut vcpus = Vec::with_capacity(config.add.into());
 
@@ -629,8 +644,13 @@ impl Vmm {
                     .vcpus_exit_evt
                     .try_clone()
                     .map_err(HotplugVcpuError::EventFd)?;
-                let vcpu =
+                let mut vcpu =
                     Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
+                if let Some(kvm_config) = self.vcpu_config.as_ref() {
+                    vcpu.kvm_vcpu.hotplug_configure(kvm_config)?;
+                } else {
+                    return Err(HotplugVcpuError::RestoredFromSnapshot);
+                }
                 locked_container.cpu_devices[cpu_idx as usize].inserting = true;
                 vcpus.push(vcpu);
             }
@@ -659,6 +679,8 @@ impl Vmm {
             huge_pages: None,
         };
 
+        self.resume_vcpu_threads(start_idx.into())?;
+
         self.acpi_device_manager.notify_cpu_container()?;
 
         Ok(new_machine_config)
@@ -871,6 +893,13 @@ impl Vmm {
         }
     }
 
+    /// Add the vcpu configuration used during boot to the VMM. This is required as part of the
+    /// hotplugging process, to correctly configure KVM vCPUs.
+    #[cfg(target_arch = "x86_64")]
+    pub fn attach_vcpu_config(&mut self, vcpu_config: VcpuConfig) {
+        self.vcpu_config = Some(vcpu_config)
+    }
+
     /// Signals Vmm to stop and exit.
     pub fn stop(&mut self, exit_code: FcExitCode) {
         // To avoid cycles, all teardown paths take the following route:

src/vmm/src/rpc_interface.rs

@@ -2149,15 +2149,31 @@ mod tests {
     #[test]
     #[cfg(target_arch = "x86_64")]
     fn test_runtime_hotplug_vcpu() {
+        use std::collections::BTreeMap;
+
+        use crate::cpu_config::x86_64::cpuid::Cpuid;
+
         // Case 1. Valid input
         let mut vmm = default_vmm();
+        let cpuid = Cpuid::try_from(vmm.vm.supported_cpuid().clone()).unwrap();
+        let vcpu_config = crate::VcpuConfig {
+            vcpu_count: 0,
+            smt: false,
+            cpu_config: crate::cpu_config::templates::CpuConfiguration {
+                cpuid,
+                msrs: BTreeMap::new(),
+            },
+        };
+
+        vmm.attach_vcpu_config(vcpu_config.clone());
         let config = HotplugVcpuConfig { add: 4 };
         let result = vmm.hotplug_vcpus(config);
         assert_eq!(vmm.vcpus_handles.len(), 4);
         result.unwrap();
 
         // Case 2. Vcpu count too low
         let mut vmm = default_vmm();
+        vmm.attach_vcpu_config(vcpu_config.clone());
         vmm.hotplug_vcpus(HotplugVcpuConfig { add: 1 }).unwrap();
         assert_eq!(vmm.vcpus_handles.len(), 1);
         let config = HotplugVcpuConfig { add: 0 };
@@ -2167,6 +2183,7 @@ mod tests {
 
         // Case 3. Vcpu count too high
         let mut vmm = default_vmm();
+        vmm.attach_vcpu_config(vcpu_config.clone());
         vmm.hotplug_vcpus(HotplugVcpuConfig { add: 1 }).unwrap();
         assert_eq!(vmm.vcpus_handles.len(), 1);
         let config = HotplugVcpuConfig { add: 33 };
@@ -2176,6 +2193,7 @@ mod tests {
 
         // Case 4. Attempted overflow of vcpus
         let mut vmm = default_vmm();
+        vmm.attach_vcpu_config(vcpu_config.clone());
         vmm.hotplug_vcpus(HotplugVcpuConfig { add: 2 }).unwrap();
         assert_eq!(vmm.vcpus_handles.len(), 2);
         let config = HotplugVcpuConfig { add: 255 };

src/vmm/src/vmm_config/hotplug.rs

@@ -5,8 +5,8 @@ use std::io;
 
 use serde::{Deserialize, Serialize};
 
-use crate::vstate::vcpu::VcpuError;
-use crate::StartVcpusError;
+use crate::vstate::vcpu::{KvmVcpuConfigureError, VcpuError};
+use crate::{StartVcpusError, VmmError};
 /// Unifying enum for all types of hotplug request configs.
 /// Currently only Vcpus may be hotplugged.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -34,10 +34,14 @@ pub enum HotplugVcpuError {
     EventFd(#[from] io::Error),
     /// Error creating the vcpu: {0}
     VcpuCreate(VcpuError),
+    /// Error configuring KVM vcpu: {0}
+    VcpuConfigure(#[from] KvmVcpuConfigureError),
     /// Failed to start vCPUs
     VcpuStart(StartVcpusError),
     /// No seccomp filter for thread category: {0}
     MissingSeccompFilters(String),
+    /// Error resuming VM: {0}
+    VmResume(#[from] VmmError),
     /// Cannot hotplug vCPUs after restoring from snapshot
     RestoredFromSnapshot,
 }

src/vmm/src/vstate/vcpu/mod.rs

@@ -63,7 +63,7 @@ pub enum VcpuError {
 }
 
 /// Encapsulates configuration parameters for the guest vCPUS.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct VcpuConfig {
     /// Number of guest VCPUs.
     pub vcpu_count: u8,

src/vmm/src/vstate/vcpu/x86_64.rs

@@ -9,8 +9,8 @@ use std::collections::BTreeMap;
 use std::fmt::Debug;
 
 use kvm_bindings::{
-    kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs,
-    kvm_xsave, CpuId, Msrs, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES,
+    kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_msr_entry, kvm_regs, kvm_sregs,
+    kvm_vcpu_events, kvm_xcrs, kvm_xsave, CpuId, Msrs, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES,
 };
 use kvm_ioctls::{VcpuExit, VcpuFd};
 use log::{error, warn};
@@ -178,22 +178,17 @@ impl KvmVcpu {
         })
     }
 
-    /// Configures a x86_64 specific vcpu for booting Linux and should be called once per vcpu.
-    ///
+    /// General configuration - common for both boot and hotplugged CPUs.
+    /// Normalizes and sets the CPUID in KVM and creates KVM MSRs
     /// # Arguments
-    ///
-    /// * `guest_mem` - The guest memory used by this microvm.
-    /// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts.
-    /// * `vcpu_config` - The vCPU configuration.
-    /// * `cpuid` - The capabilities exposed by this vCPU.
-    pub fn configure(
+    /// * vcpu_config - The configuration for the vCPUs.
+    /// * msrs - The MSRs currently present.
+    fn configure_common(
         &mut self,
-        guest_mem: &GuestMemoryMmap,
-        kernel_start_addr: GuestAddress,
         vcpu_config: &VcpuConfig,
-    ) -> Result<(), KvmVcpuConfigureError> {
+        msrs: std::collections::BTreeMap<u32, u64>,
+    ) -> Result<(Vec<kvm_msr_entry>, CpuId), KvmVcpuConfigureError> {
         let mut cpuid = vcpu_config.cpu_config.cpuid.clone();
-
         // Apply machine specific changes to CPUID.
         cpuid.normalize(
             // The index of the current logical CPU in the range [0..cpu_count].
@@ -212,6 +207,35 @@ impl KvmVcpu {
             .set_cpuid2(&kvm_cpuid)
             .map_err(KvmVcpuConfigureError::SetCpuid)?;
 
+        // // Clone MSR entries that are modified by CPU template from `VcpuConfig`.
+
+        let kvm_msrs = msrs
+            .clone()
+            .into_iter()
+            .map(|entry| kvm_bindings::kvm_msr_entry {
+                index: entry.0,
+                data: entry.1,
+                ..Default::default()
+            })
+            .collect::<Vec<_>>();
+
+        Ok((kvm_msrs, kvm_cpuid))
+    }
+
+    /// Configures a x86_64 specific vcpu for booting Linux and should be called once per vcpu.
+    ///
+    /// # Arguments
+    ///
+    /// * `guest_mem` - The guest memory used by this microvm.
+    /// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts.
+    /// * `vcpu_config` - The vCPU configuration.
+    /// * `cpuid` - The capabilities exposed by this vCPU.
+    pub fn configure(
+        &mut self,
+        guest_mem: &GuestMemoryMmap,
+        kernel_start_addr: GuestAddress,
+        vcpu_config: &VcpuConfig,
+    ) -> Result<(), KvmVcpuConfigureError> {
         // Clone MSR entries that are modified by CPU template from `VcpuConfig`.
         let mut msrs = vcpu_config.cpu_config.msrs.clone();
         self.msrs_to_save.extend(msrs.keys());
@@ -221,6 +245,8 @@ impl KvmVcpu {
             msrs.insert(entry.index, entry.data);
         });
 
+        let (kvm_msrs, kvm_cpuid) = self.configure_common(vcpu_config, msrs)?;
+
         // TODO - Add/amend MSRs for vCPUs based on cpu_config
         // By this point the Guest CPUID is established. Some CPU features require MSRs
         // to configure and interact with those features. If a MSR is writable from
@@ -239,15 +265,6 @@ impl KvmVcpu {
         // save is `architectural MSRs` + `MSRs inferred through CPUID` + `other
         // MSRs defined by the template`
 
-        let kvm_msrs = msrs
-            .into_iter()
-            .map(|entry| kvm_bindings::kvm_msr_entry {
-                index: entry.0,
-                data: entry.1,
-                ..Default::default()
-            })
-            .collect::<Vec<_>>();
-
         crate::arch::x86_64::msr::set_msrs(&self.fd, &kvm_msrs)?;
         crate::arch::x86_64::regs::setup_regs(&self.fd, kernel_start_addr.raw_value())?;
         crate::arch::x86_64::regs::setup_fpu(&self.fd)?;
@@ -257,6 +274,25 @@ impl KvmVcpu {
         Ok(())
     }
 
+    /// Configures an x86_64 cpu that has been hotplugged post-boot. Called once per hotplugged
+    /// vcpu.
+    ///
+    /// # Arguments
+    ///
+    /// * `vcpu_config` - The config to be applied to the vCPU, defined at boot time.
+    pub fn hotplug_configure(
+        &mut self,
+        vcpu_config: &VcpuConfig,
+    ) -> Result<(), KvmVcpuConfigureError> {
+        let msrs = vcpu_config.cpu_config.msrs.clone();
+        let (kvm_msrs, _) = self.configure_common(vcpu_config, msrs)?;
+
+        crate::arch::x86_64::msr::set_msrs(&self.fd, &kvm_msrs)?;
+        crate::arch::x86_64::interrupts::set_lint(&self.fd)?;
+
+        Ok(())
+    }
+
     /// Sets a Port Mapped IO bus for this vcpu.
     pub fn set_pio_bus(&mut self, pio_bus: crate::devices::Bus) {
         self.peripherals.pio_bus = Some(pio_bus);