MMIO Portion of hotplugging

Implement MMIO for the CpuContainer. This allows the AML methods to read
from and write to the defined AML fields.

Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>

Author: JamesC1305

src/vmm/src/arch/mod.rs

@@ -41,6 +41,9 @@ pub enum DeviceType {
     Rtc,
     /// Device Type: BootTimer.
     BootTimer,
+    /// Device Type: CpuContainer
+    #[cfg(target_arch = "x86_64")]
+    CpuContainer,
 }
 
 /// Type for passing information about the initrd in the guest memory.

src/vmm/src/builder.rs

@@ -1028,8 +1028,16 @@ fn attach_cpu_container_device(vmm: &mut Vmm, vcpu_count: u8) -> Result<(), Star
         vcpu_count,
     )?));
     vmm.acpi_device_manager
-        .attach_cpu_container(cpu_container, vmm.vm.fd())
+        .attach_cpu_container(cpu_container.clone(), vmm.vm.fd())
         .map_err(StartMicrovmError::AttachCpuContainerDevice)?;
+    vmm.mmio_device_manager
+        .register_mmio_cpu_container_for_boot(
+            vmm.vm.fd(),
+            &mut vmm.resource_allocator,
+            cpu_container,
+        )
+        .map_err(StartMicrovmError::RegisterMmioDevice)?;
+
     Ok(())
 }
 
@@ -1143,6 +1151,9 @@ pub mod tests {
         #[cfg(target_arch = "aarch64")]
         let resource_allocator = ResourceAllocator::new().unwrap();
 
+        #[cfg(target_arch = "x86_64")]
+        let mut mmio_device_manager = MMIODeviceManager::new();
+        #[cfg(target_arch = "aarch64")]
         let mmio_device_manager = MMIODeviceManager::new();
 
         #[cfg(target_arch = "x86_64")]
@@ -1166,16 +1177,21 @@ pub mod tests {
         )
         .unwrap();
 
-        #[cfg(target_arch = "x86_64")]
-        setup_interrupt_controller(&mut vm).unwrap();
-
         #[cfg(target_arch = "x86_64")]
         {
+            setup_interrupt_controller(&mut vm).unwrap();
             let cpu_container = Arc::new(Mutex::new(
                 CpuContainer::new(&mut resource_allocator, 1).unwrap(),
             ));
             acpi_device_manager
-                .attach_cpu_container(cpu_container, vm.fd())
+                .attach_cpu_container(cpu_container.clone(), vm.fd())
+                .unwrap();
+            mmio_device_manager
+                .register_mmio_cpu_container_for_boot(
+                    vm.fd(),
+                    &mut resource_allocator,
+                    cpu_container,
+                )
                 .unwrap();
         }
 
@@ -1559,8 +1575,8 @@ pub mod tests {
             #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
             assert!(cmdline_contains(
                 &cmdline,
-                "virtio_mmio.device=4K@0xd0001000:6 virtio_mmio.device=4K@0xd0002000:7 \
-                 virtio_mmio.device=4K@0xd0003000:8"
+                "virtio_mmio.device=4K@0xd0001000:7 virtio_mmio.device=4K@0xd0002000:8 \
+                 virtio_mmio.device=4K@0xd0003000:9"
             ));
         }
 
@@ -1655,7 +1671,7 @@ pub mod tests {
         #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
         assert!(cmdline_contains(
             &cmdline,
-            "virtio_mmio.device=4K@0xd0001000:6"
+            "virtio_mmio.device=4K@0xd0001000:7"
         ));
     }
 
@@ -1672,7 +1688,7 @@ pub mod tests {
         #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
         assert!(cmdline_contains(
             &cmdline,
-            "virtio_mmio.device=4K@0xd0001000:6"
+            "virtio_mmio.device=4K@0xd0001000:7"
         ));
     }
 
@@ -1691,7 +1707,7 @@ pub mod tests {
         #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
         assert!(cmdline_contains(
             &cmdline,
-            "virtio_mmio.device=4K@0xd0001000:6"
+            "virtio_mmio.device=4K@0xd0001000:7"
         ));
     }
 }

src/vmm/src/device_manager/mmio.rs

@@ -24,6 +24,8 @@ use super::resources::ResourceAllocator;
 use crate::arch::aarch64::DeviceInfoForFDT;
 use crate::arch::DeviceType;
 use crate::arch::DeviceType::Virtio;
+#[cfg(target_arch = "x86_64")]
+use crate::devices::acpi::cpu_container::CpuContainer;
 #[cfg(target_arch = "aarch64")]
 use crate::devices::legacy::{RTCDevice, SerialDevice};
 use crate::devices::pseudo::BootTimer;
@@ -343,6 +345,52 @@ impl MMIODeviceManager {
         )
     }
 
+    #[cfg(target_arch = "x86_64")]
+    pub fn register_mmio_cpu_container(
+        &mut self,
+        vm: &VmFd,
+        device: Arc<Mutex<CpuContainer>>,
+        device_info: &MMIODeviceInfo,
+    ) -> Result<(), MmioError> {
+        let identifier = (
+            DeviceType::CpuContainer,
+            DeviceType::CpuContainer.to_string(),
+        );
+        {
+            let container = device.lock().expect("Poisoned lock");
+            vm.register_irqfd(&container.mmio_interrupt_evt, device_info.irqs[0])
+                .map_err(MmioError::RegisterIrqFd)?;
+        }
+
+        self.register_mmio_device(
+            identifier,
+            device_info.clone(),
+            BusDevice::CpuContainer(device),
+        )?;
+        Ok(())
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn register_mmio_cpu_container_for_boot(
+        &mut self,
+        vm: &VmFd,
+        resource_allocator: &mut ResourceAllocator,
+        device: Arc<Mutex<CpuContainer>>,
+    ) -> Result<(), MmioError> {
+        let device_info = {
+            let locked_container = device.lock().expect("Poisoned lock");
+            let irqs = resource_allocator.allocate_gsi(1)?;
+            MMIODeviceInfo {
+                addr: locked_container.mmio_address.0,
+                len: MMIO_LEN,
+                irqs,
+            }
+        };
+
+        self.register_mmio_cpu_container(vm, device, &device_info)?;
+        Ok(())
+    }
+
     /// Gets the information of the devices registered up to some point in time.
     pub fn get_device_info(&self) -> &HashMap<(DeviceType, String), MMIODeviceInfo> {
         &self.id_to_dev_info
@@ -831,4 +879,24 @@ mod tests {
             .allocate_mmio_resources(&mut resource_allocator, 0)
             .unwrap();
     }
+
+    #[test]
+    #[cfg(target_arch = "x86_64")]
+    fn test_register_cpu_container() {
+        let mut device_manager = MMIODeviceManager::new();
+        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let mut vm = Vm::new(vec![]).unwrap();
+        builder::setup_interrupt_controller(&mut vm).unwrap();
+        let cpu_container = Arc::new(Mutex::new(
+            CpuContainer::new(&mut resource_allocator, 1).unwrap(),
+        ));
+
+        device_manager
+            .register_mmio_cpu_container_for_boot(vm.fd(), &mut resource_allocator, cpu_container)
+            .unwrap();
+
+        device_manager
+            .get_device(DeviceType::CpuContainer, "CpuContainer")
+            .unwrap();
+    }
 }

src/vmm/src/devices/acpi/cpu_container.rs

@@ -41,6 +41,8 @@ pub enum CpuContainerError {
     GuestMemory(#[from] GuestMemoryError),
     /// Failed to allocate requested resource: {0}
     Allocator(#[from] vm_allocator::Error),
+    /// Failed to register file descriptor: {0}
+    RegisterIrqFd(#[from] kvm_ioctls::Error),
 }
 
 pub const CPU_CONTAINER_ACPI_SIZE: usize = 0xC;
@@ -78,7 +80,7 @@ impl CpuContainer {
         boot_count: u8,
     ) -> Result<Self, CpuContainerError> {
         let gsi = resource_allocator.allocate_gsi(1)?;
-        let mmio_address = resource_allocator.allocate_system_memory(
+        let mmio_address = resource_allocator.allocate_mmio_memory(
             MMIO_LEN,
             MMIO_LEN,
             vm_allocator::AllocPolicy::FirstMatch,
@@ -103,6 +105,49 @@ impl CpuContainer {
         debug!("hotplug: notifying guest about new vcpus available");
         Ok(())
     }
+
+    pub fn bus_read(&mut self, offset: u64, data: &mut [u8]) {
+        data.fill(0);
+        match offset {
+            CPU_SELECTION_OFFSET => {
+                data[0] = self.selected_cpu;
+            }
+            CPU_STATUS_OFFSET => {
+                if self.selected_cpu < MAX_SUPPORTED_VCPUS {
+                    let cpu_device = &self.cpu_devices[self.selected_cpu as usize];
+                    if cpu_device.active {
+                        data[0] |= CPU_ENABLE_BIT;
+                    }
+                    if cpu_device.inserting {
+                        data[0] |= CPU_INSERTING_BIT;
+                    }
+                } else {
+                    error!("Out of range vCPU id: {}", self.selected_cpu)
+                }
+            }
+            _ => error!("Unexpected CPU container offset"),
+        }
+    }
+
+    pub fn bus_write(&mut self, offset: u64, data: &[u8]) {
+        match offset {
+            CPU_SELECTION_OFFSET => self.selected_cpu = data[0],
+            CPU_STATUS_OFFSET => {
+                if self.selected_cpu < MAX_SUPPORTED_VCPUS {
+                    let cpu_device = &mut self.cpu_devices[self.selected_cpu as usize];
+                    if data[0] & CPU_INSERTING_BIT != 0 {
+                        cpu_device.inserting = false;
+                    }
+                    if data[0] & CPU_ENABLE_BIT != 0 {
+                        cpu_device.active = true;
+                    }
+                } else {
+                    error!("Out of range vCPU id: {}", self.selected_cpu)
+                }
+            }
+            _ => error!("Unexpected CPU container offset"),
+        }
+    }
 }
 
 impl Aml for CpuContainer {

src/vmm/src/devices/bus.rs

@@ -52,6 +52,8 @@ pub struct Bus {
 
 use event_manager::{EventOps, Events, MutEventSubscriber};
 
+#[cfg(target_arch = "x86_64")]
+use super::acpi::cpu_container::CpuContainer;
 #[cfg(target_arch = "aarch64")]
 use super::legacy::RTCDevice;
 use super::legacy::{I8042Device, SerialDevice};
@@ -66,6 +68,8 @@ pub enum BusDevice {
     BootTimer(Arc<Mutex<BootTimer>>),
     MmioTransport(Arc<Mutex<MmioTransport>>),
     Serial(Arc<Mutex<SerialDevice<std::io::Stdin>>>),
+    #[cfg(target_arch = "x86_64")]
+    CpuContainer(Arc<Mutex<CpuContainer>>),
     #[cfg(test)]
     Dummy(Arc<Mutex<DummyDevice>>),
     #[cfg(test)]
@@ -130,6 +134,8 @@ impl BusDevice {
             Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
             Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
             Self::Serial(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
+            #[cfg(target_arch = "x86_64")]
+            Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
             #[cfg(test)]
             Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
             #[cfg(test)]
@@ -145,6 +151,8 @@ impl BusDevice {
             Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
             Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
             Self::Serial(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
+            #[cfg(target_arch = "x86_64")]
+            Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
             #[cfg(test)]
             Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
             #[cfg(test)]

src/vmm/src/lib.rs

@@ -620,14 +620,20 @@ impl Vmm {
 
         #[allow(clippy::cast_possible_truncation)]
         let start_idx = self.vcpus_handles.len().try_into().unwrap();
-        for cpu_idx in start_idx..(start_idx + config.add) {
-            let exit_evt = self
-                .vcpus_exit_evt
-                .try_clone()
-                .map_err(HotplugVcpuError::EventFd)?;
-            let vcpu =
-                Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
-            vcpus.push(vcpu);
+        if let Some(devices::BusDevice::CpuContainer(cont)) =
+            self.get_bus_device(DeviceType::CpuContainer, "CpuContainer")
+        {
+            let mut locked_container = cont.lock().expect("Poisoned lock");
+            for cpu_idx in start_idx..(start_idx + config.add) {
+                let exit_evt = self
+                    .vcpus_exit_evt
+                    .try_clone()
+                    .map_err(HotplugVcpuError::EventFd)?;
+                let vcpu =
+                    Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
+                locked_container.cpu_devices[cpu_idx as usize].inserting = true;
+                vcpus.push(vcpu);
+            }
         }
 
         self.start_vcpus(
@@ -653,6 +659,8 @@ impl Vmm {
             huge_pages: None,
         };
 
+        self.acpi_device_manager.notify_cpu_container()?;
+
         Ok(new_machine_config)
     }
 

tests/integration_tests/functional/test_max_devices.py

@@ -6,9 +6,10 @@
 
 import pytest
 
-# IRQs are available from 5 to 23. We always use one IRQ for VMGenID device, so
-# the maximum number of devices supported at the same time is 18.
-MAX_DEVICES_ATTACHED = 18
+# IRQs are available from 5 to 23. We always use one IRQ for VMGenID device and
+# two for CPU hotplugging, so the maximum number of devices supported at the same
+# time is 16.
+MAX_DEVICES_ATTACHED = 16
 
 
 @pytest.mark.skipif(