MMIO Portion of hotplugging

Implement MMIO for the CpuContainer. This allows the AML methods to read
from and write to the defined AML fields.

Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>

Author: JamesC1305

src/vmm/src/arch/mod.rs

@@ -41,6 +41,9 @@ pub enum DeviceType {
     Rtc,
     /// Device Type: BootTimer.
     BootTimer,
+    /// Device Type: CpuContainer
+    #[cfg(target_arch = "x86_64")]
+    CpuContainer,
 }
 
 /// Type for passing information about the initrd in the guest memory.

src/vmm/src/builder.rs

@@ -38,6 +38,7 @@ use crate::cpu_config::templates::{
     CpuConfiguration, CustomCpuTemplate, GetCpuTemplate, GetCpuTemplateError, GuestConfigError,
     KvmCapability,
 };
+use crate::cpu_config::x86_64::cpuid::{self, Cpuid};
 #[cfg(target_arch = "x86_64")]
 use crate::device_manager::acpi::ACPIDeviceManager;
 #[cfg(target_arch = "x86_64")]
@@ -86,6 +87,9 @@ pub enum StartMicrovmError {
     /// Unable to attach the VMGenID device: {0}
     #[cfg(target_arch = "x86_64")]
     AttachVmgenidDevice(kvm_ioctls::Error),
+    /// Unable to attach the CpuContainer device: {0}
+    #[cfg(target_arch = "x86_64")]
+    AttachCpuContainerDevice(kvm_ioctls::Error),
     /// System configuration error: {0}
     ConfigureSystem(crate::arch::ConfigurationError),
     /// Failed to create guest config: {0}
@@ -185,18 +189,32 @@ fn create_vmm_and_vcpus(
     // Instantiate the MMIO device manager.
     let mut mmio_device_manager = MMIODeviceManager::new();
 
-    if boot_timer_enabled {
-        let boot_timer = crate::devices::pseudo::BootTimer::new(TimestampUs::default());
+    #[cfg(target_arch = "x86_64")]
+    {
+        // For x86, we need to create the interrupt controller before calling `KVM_CREATE_VCPUS`,
+        // but we also need it before the instantiation of the ACPI Device manager,
+        // this is because the CpuContainer needs to create and register IRQs
+        setup_interrupt_controller(&mut vm)?;
 
-        mmio_device_manager
-            .register_mmio_boot_timer(&mut resource_allocator, boot_timer)
-            .map_err(RegisterMmioDevice)?;
+        // The boot timer device needs to be the first device attached in order
+        // to maintain the same MMIO address referenced in the documentation
+        // and tests.
+        // This has to instantiated here, before the CpuContainer, to ensure that it gets the
+        // correct address, the first page of MMIO memory.
+        if boot_timer_enabled {
+            let boot_timer = crate::devices::pseudo::BootTimer::new(TimestampUs::default());
+
+            mmio_device_manager
+                .register_mmio_boot_timer(&mut resource_allocator, boot_timer)
+                .map_err(RegisterMmioDevice)?;
+        }
     }
 
     // Instantiate ACPI device manager.
     #[cfg(target_arch = "x86_64")]
     let acpi_device_manager = {
         let cpu_container = Arc::new(Mutex::new(CpuContainer::new(
+            vm.fd(),
             &mut resource_allocator,
             vcpu_count,
         )?));
@@ -207,7 +225,6 @@ fn create_vmm_and_vcpus(
     // while on aarch64 we need to do it the other way around.
     #[cfg(target_arch = "x86_64")]
     let (vcpus, pio_device_manager) = {
-        setup_interrupt_controller(&mut vm)?;
         let vcpus = create_vcpus(&vm, vcpu_count, &vcpus_exit_evt).map_err(Internal)?;
 
         // Make stdout non blocking.
@@ -281,7 +298,7 @@ pub fn build_microvm_for_boot(
     use self::StartMicrovmError::*;
 
     // Timestamp for measuring microVM boot duration.
-    let request_ts = TimestampUs::default();
+    // let request_ts = TimestampUs::default();
 
     let boot_config = vm_resources
         .boot_source_builder()
@@ -342,13 +359,6 @@ pub fn build_microvm_for_boot(
         vm_resources.boot_timer,
     )?;
 
-    // The boot timer device needs to be the first device attached in order
-    // to maintain the same MMIO address referenced in the documentation
-    // and tests.
-    // if vm_resources.boot_timer {
-    //     attach_boot_timer_device(&mut vmm, request_ts)?;
-    // }
-
     #[cfg(target_arch = "x86_64")]
     attach_cpu_container_device(&mut vmm)?;
 
@@ -570,14 +580,20 @@ pub fn build_microvm_from_snapshot(
 
     #[cfg(target_arch = "x86_64")]
     {
-        let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
-            mem: &guest_memory,
-            resource_allocator: &mut vmm.resource_allocator,
-            vm: vmm.vm.fd(),
-        };
+        if let Some(BusDevice::CpuContainer(container)) =
+            vmm.get_bus_device(DeviceType::CpuContainer, "CpuContainer")
+        {
+            let container_ref = container.clone();
+            let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
+                mem: &guest_memory,
+                resource_allocator: &mut vmm.resource_allocator,
+                vm: vmm.vm.fd(),
+                cpu_container: container_ref,
+            };
 
-        vmm.acpi_device_manager =
-            ACPIDeviceManager::restore(acpi_ctor_args, &microvm_state.acpi_dev_state)?;
+            vmm.acpi_device_manager =
+                ACPIDeviceManager::restore(acpi_ctor_args, &microvm_state.acpi_dev_state)?;
+        }
 
         // Inject the notification to VMGenID that we have resumed from a snapshot.
         // This needs to happen before we resume vCPUs, so that we minimize the time between vCPUs
@@ -930,6 +946,8 @@ fn attach_virtio_device<T: 'static + VirtioDevice + MutEventSubscriber + Debug>(
         .map(|_| ())
 }
 
+// Temporarily test only. (waiting for decision on CpuContainer being an option or not)
+#[cfg(target_arch = "x86_64")]
 pub(crate) fn attach_boot_timer_device(
     vmm: &mut Vmm,
     request_ts: TimestampUs,
@@ -1051,6 +1069,20 @@ fn attach_balloon_device(
     attach_virtio_device(event_manager, vmm, id, balloon.clone(), cmdline, false)
 }
 
+#[cfg(target_arch = "x86_64")]
+fn attach_cpu_container_device(vmm: &mut Vmm) -> Result<(), StartMicrovmError> {
+    let cpu_container = vmm.acpi_device_manager.cpu_container.clone();
+    vmm.mmio_device_manager
+        .register_mmio_cpu_container_for_boot(
+            vmm.vm.fd(),
+            &mut vmm.resource_allocator,
+            cpu_container,
+        )
+        .map_err(StartMicrovmError::RegisterMmioDevice)?;
+
+    Ok(())
+}
+
 // Adds `O_NONBLOCK` to the stdout flags.
 pub(crate) fn set_stdout_nonblocking() {
     // SAFETY: Call is safe since parameters are valid.
@@ -1161,6 +1193,9 @@ pub mod tests {
         #[cfg(target_arch = "aarch64")]
         let resource_allocator = ResourceAllocator::new().unwrap();
 
+        #[cfg(target_arch = "x86_64")]
+        let mut mmio_device_manager = MMIODeviceManager::new();
+        #[cfg(target_arch = "aarch64")]
         let mmio_device_manager = MMIODeviceManager::new();
 
         #[cfg(target_arch = "x86_64")]
@@ -1169,11 +1204,11 @@ pub mod tests {
         #[cfg(target_arch = "x86_64")]
         let acpi_device_manager = {
             let cpu_container = Arc::new(Mutex::new(
-                CpuContainer::new(&mut resource_allocator, 1)
+                CpuContainer::new(vm.fd(), &mut resource_allocator, 1)
                     .map_err(StartMicrovmError::CreateCpuContainer)
                     .unwrap(),
             ));
-            ACPIDeviceManager::new(cpu_container)
+            ACPIDeviceManager::new(cpu_container.clone())
         };
 
         #[cfg(target_arch = "x86_64")]
@@ -1199,6 +1234,15 @@ pub mod tests {
             setup_interrupt_controller(&mut vm, 1).unwrap();
         }
 
+        #[cfg(target_arch = "x86_64")]
+        mmio_device_manager
+            .register_mmio_cpu_container_for_boot(
+                vm.fd(),
+                &mut resource_allocator,
+                acpi_device_manager.cpu_container.clone(),
+            )
+            .unwrap();
+
         Vmm {
             events_observer: Some(std::io::stdin()),
             instance_info: InstanceInfo::default(),

src/vmm/src/device_manager/mmio.rs

@@ -24,6 +24,8 @@ use super::resources::ResourceAllocator;
 use crate::arch::aarch64::DeviceInfoForFDT;
 use crate::arch::DeviceType;
 use crate::arch::DeviceType::Virtio;
+#[cfg(target_arch = "x86_64")]
+use crate::devices::acpi::cpu_container::CpuContainer;
 #[cfg(target_arch = "aarch64")]
 use crate::devices::legacy::{RTCDevice, SerialDevice};
 use crate::devices::pseudo::BootTimer;
@@ -319,7 +321,7 @@ impl MMIODeviceManager {
         self.register_mmio_device(
             identifier,
             device_info,
-            Arc::new(Mutex::new(BusDevice::RTCDevice(rtc))),
+            BusDevice::RTCDevice(Arc::new(Mutex::new(rtc))),
         )
     }
 
@@ -340,6 +342,52 @@ impl MMIODeviceManager {
         )
     }
 
+    #[cfg(target_arch = "x86_64")]
+    pub fn register_mmio_cpu_container(
+        &mut self,
+        vm: &VmFd,
+        device: Arc<Mutex<CpuContainer>>,
+        device_info: &MMIODeviceInfo,
+    ) -> Result<(), MmioError> {
+        let identifier = (
+            DeviceType::CpuContainer,
+            DeviceType::CpuContainer.to_string(),
+        );
+        {
+            let container = device.lock().expect("Poisoned lock");
+            vm.register_irqfd(&container.mmio_interrupt_evt, device_info.irqs[0])
+                .map_err(MmioError::RegisterIrqFd)?;
+        }
+
+        self.register_mmio_device(
+            identifier,
+            device_info.clone(),
+            BusDevice::CpuContainer(device),
+        )?;
+        Ok(())
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn register_mmio_cpu_container_for_boot(
+        &mut self,
+        vm: &VmFd,
+        resource_allocator: &mut ResourceAllocator,
+        device: Arc<Mutex<CpuContainer>>,
+    ) -> Result<(), MmioError> {
+        let device_info = {
+            let locked_container = device.lock().expect("Poisoned lock");
+            let irqs = resource_allocator.allocate_gsi(1)?;
+            MMIODeviceInfo {
+                addr: locked_container.mmio_address.0,
+                len: MMIO_LEN,
+                irqs,
+            }
+        };
+
+        self.register_mmio_cpu_container(vm, device, &device_info)?;
+        Ok(())
+    }
+
     /// Gets the information of the devices registered up to some point in time.
     pub fn get_device_info(&self) -> &HashMap<(DeviceType, String), MMIODeviceInfo> {
         &self.id_to_dev_info
@@ -832,4 +880,24 @@ mod tests {
             .allocate_mmio_resources(&mut resource_allocator, 0)
             .unwrap();
     }
+
+    #[test]
+    #[cfg(target_arch = "x86_64")]
+    fn test_register_cpu_container() {
+        let mut device_manager = MMIODeviceManager::new();
+        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let mut vm = Vm::new(vec![]).unwrap();
+        builder::setup_interrupt_controller(&mut vm).unwrap();
+        let cpu_container = Arc::new(Mutex::new(
+            CpuContainer::new(vm.fd(), &mut resource_allocator, 1).unwrap(),
+        ));
+
+        device_manager
+            .register_mmio_cpu_container_for_boot(vm.fd(), &mut resource_allocator, cpu_container)
+            .unwrap();
+
+        device_manager
+            .get_device(DeviceType::CpuContainer, "CpuContainer")
+            .unwrap();
+    }
 }