Skip to content

Commit ecb9e4b

Browse files
JamesC1305JamesC1305
committed
MMIO Portion of hotplugging
Implement MMIO for the CpuContainer. This allows the AML methods to read from and write to the defined AML fields. Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>
1 parent c916b68 commit ecb9e4b

File tree

8 files changed

+198
-33
lines changed

8 files changed

+198
-33
lines changed

‎src/vmm/src/arch/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ pub enum DeviceType {
4141
Rtc,
4242
/// Device Type: BootTimer.
4343
BootTimer,
44+
/// Device Type: CpuContainer
45+
#[cfg(target_arch = "x86_64")]
46+
CpuContainer,
4447
}
4548

4649
/// Type for passing information about the initrd in the guest memory.

‎src/vmm/src/builder.rs‎

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ use crate::cpu_config::templates::{
3838
CpuConfiguration, CustomCpuTemplate, GetCpuTemplate, GetCpuTemplateError, GuestConfigError,
3939
KvmCapability,
4040
};
41+
use crate::cpu_config::x86_64::cpuid::{self, Cpuid};
4142
#[cfg(target_arch = "x86_64")]
4243
use crate::device_manager::acpi::ACPIDeviceManager;
4344
#[cfg(target_arch = "x86_64")]
@@ -86,6 +87,9 @@ pub enum StartMicrovmError {
8687
/// Unable to attach the VMGenID device: {0}
8788
#[cfg(target_arch = "x86_64")]
8889
AttachVmgenidDevice(kvm_ioctls::Error),
90+
/// Unable to attach the CpuContainer device: {0}
91+
#[cfg(target_arch = "x86_64")]
92+
AttachCpuContainerDevice(kvm_ioctls::Error),
8993
/// System configuration error: {0}
9094
ConfigureSystem(crate::arch::ConfigurationError),
9195
/// Failed to create guest config: {0}
@@ -179,10 +183,10 @@ fn create_vmm_and_vcpus(
179183
.map_err(VmmError::EventFd)
180184
.map_err(Internal)?;
181185

182-
let mut resource_allocator = ResourceAllocator::new()?;
186+
let resource_allocator = ResourceAllocator::new()?;
183187

184188
// Instantiate the MMIO device manager.
185-
let mut mmio_device_manager = MMIODeviceManager::new();
189+
let mmio_device_manager = MMIODeviceManager::new();
186190

187191
// Instantiate ACPI device manager.
188192
#[cfg(target_arch = "x86_64")]
@@ -553,14 +557,20 @@ pub fn build_microvm_from_snapshot(
553557

554558
#[cfg(target_arch = "x86_64")]
555559
{
556-
let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
557-
mem: &guest_memory,
558-
resource_allocator: &mut vmm.resource_allocator,
559-
vm: vmm.vm.fd(),
560-
};
560+
if let Some(BusDevice::CpuContainer(container)) =
561+
vmm.get_bus_device(DeviceType::CpuContainer, "CpuContainer")
562+
{
563+
let container_ref = container.clone();
564+
let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
565+
mem: &guest_memory,
566+
resource_allocator: &mut vmm.resource_allocator,
567+
vm: vmm.vm.fd(),
568+
cpu_container: container_ref,
569+
};
561570

562-
vmm.acpi_device_manager =
563-
ACPIDeviceManager::restore(acpi_ctor_args, &microvm_state.acpi_dev_state)?;
571+
vmm.acpi_device_manager =
572+
ACPIDeviceManager::restore(acpi_ctor_args, &microvm_state.acpi_dev_state)?;
573+
}
564574

565575
// Inject the notification to VMGenID that we have resumed from a snapshot.
566576
// This needs to happen before we resume vCPUs, so that we minimize the time between vCPUs
@@ -1040,7 +1050,16 @@ fn attach_cpu_container_device(vmm: &mut Vmm, vcpu_count: u8) -> Result<(), Star
10401050
vcpu_count,
10411051
)?));
10421052
vmm.acpi_device_manager
1043-
.attach_cpu_container(cpu_container.clone(), vmm.vm.fd());
1053+
.attach_cpu_container(cpu_container.clone(), vmm.vm.fd())
1054+
.map_err(StartMicrovmError::AttachCpuContainerDevice)?;
1055+
vmm.mmio_device_manager
1056+
.register_mmio_cpu_container_for_boot(
1057+
vmm.vm.fd(),
1058+
&mut vmm.resource_allocator,
1059+
cpu_container,
1060+
)
1061+
.map_err(StartMicrovmError::RegisterMmioDevice)?;
1062+
10441063
Ok(())
10451064
}
10461065

@@ -1154,10 +1173,13 @@ pub mod tests {
11541173
#[cfg(target_arch = "aarch64")]
11551174
let resource_allocator = ResourceAllocator::new().unwrap();
11561175

1176+
#[cfg(target_arch = "x86_64")]
1177+
let mut mmio_device_manager = MMIODeviceManager::new();
1178+
#[cfg(target_arch = "aarch64")]
11571179
let mmio_device_manager = MMIODeviceManager::new();
11581180

11591181
#[cfg(target_arch = "x86_64")]
1160-
let acpi_device_manager = ACPIDeviceManager::new();
1182+
let mut acpi_device_manager = ACPIDeviceManager::new();
11611183

11621184
#[cfg(target_arch = "x86_64")]
11631185
let pio_device_manager = PortIODeviceManager::new(
@@ -1175,15 +1197,22 @@ pub mod tests {
11751197
)
11761198
.unwrap();
11771199

1178-
#[cfg(target_arch = "x86_64")]
1179-
setup_interrupt_controller(&mut vm).unwrap();
1180-
11811200
#[cfg(target_arch = "x86_64")]
11821201
{
1202+
setup_interrupt_controller(&mut vm).unwrap();
11831203
let cpu_container = Arc::new(Mutex::new(
11841204
CpuContainer::new(&mut resource_allocator, 1).unwrap(),
11851205
));
1186-
acpi_device_manager.attach_cpu_container(cpu_container.clone, vm.fd())
1206+
acpi_device_manager
1207+
.attach_cpu_container(cpu_container.clone(), vm.fd())
1208+
.unwrap();
1209+
mmio_device_manager
1210+
.register_mmio_cpu_container_for_boot(
1211+
vm.fd(),
1212+
&mut resource_allocator,
1213+
cpu_container,
1214+
)
1215+
.unwrap();
11871216
}
11881217

11891218
#[cfg(target_arch = "aarch64")]

‎src/vmm/src/device_manager/acpi.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ impl ACPIDeviceManager {
4343
cpu_container: Arc<Mutex<CpuContainer>>,
4444
vm_fd: &VmFd,
4545
) -> Result<(), kvm_ioctls::Error> {
46-
let locked_container = cpu_container.lock().expect("Poisoned lock");
47-
vm_fd.register_irqfd(&locked_container.acpi_interrupt_evt, locked_container.gsi)?;
46+
{
47+
let locked_container = cpu_container.lock().expect("Poisoned lock");
48+
vm_fd.register_irqfd(&locked_container.acpi_interrupt_evt, locked_container.gsi)?;
49+
}
4850
self.cpu_container = Some(cpu_container);
4951
Ok(())
5052
}

‎src/vmm/src/device_manager/mmio.rs‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ use super::resources::ResourceAllocator;
2424
use crate::arch::aarch64::DeviceInfoForFDT;
2525
use crate::arch::DeviceType;
2626
use crate::arch::DeviceType::Virtio;
27+
#[cfg(target_arch = "x86_64")]
28+
use crate::devices::acpi::cpu_container::CpuContainer;
2729
#[cfg(target_arch = "aarch64")]
2830
use crate::devices::legacy::{RTCDevice, SerialDevice};
2931
use crate::devices::pseudo::BootTimer;
@@ -319,7 +321,7 @@ impl MMIODeviceManager {
319321
self.register_mmio_device(
320322
identifier,
321323
device_info,
322-
Arc::new(Mutex::new(BusDevice::RTCDevice(rtc))),
324+
BusDevice::RTCDevice(Arc::new(Mutex::new(rtc))),
323325
)
324326
}
325327

@@ -340,6 +342,52 @@ impl MMIODeviceManager {
340342
)
341343
}
342344

345+
#[cfg(target_arch = "x86_64")]
346+
pub fn register_mmio_cpu_container(
347+
&mut self,
348+
vm: &VmFd,
349+
device: Arc<Mutex<CpuContainer>>,
350+
device_info: &MMIODeviceInfo,
351+
) -> Result<(), MmioError> {
352+
let identifier = (
353+
DeviceType::CpuContainer,
354+
DeviceType::CpuContainer.to_string(),
355+
);
356+
{
357+
let container = device.lock().expect("Poisoned lock");
358+
vm.register_irqfd(&container.mmio_interrupt_evt, device_info.irqs[0])
359+
.map_err(MmioError::RegisterIrqFd)?;
360+
}
361+
362+
self.register_mmio_device(
363+
identifier,
364+
device_info.clone(),
365+
BusDevice::CpuContainer(device),
366+
)?;
367+
Ok(())
368+
}
369+
370+
#[cfg(target_arch = "x86_64")]
371+
pub fn register_mmio_cpu_container_for_boot(
372+
&mut self,
373+
vm: &VmFd,
374+
resource_allocator: &mut ResourceAllocator,
375+
device: Arc<Mutex<CpuContainer>>,
376+
) -> Result<(), MmioError> {
377+
let device_info = {
378+
let locked_container = device.lock().expect("Poisoned lock");
379+
let irqs = resource_allocator.allocate_gsi(1)?;
380+
MMIODeviceInfo {
381+
addr: locked_container.mmio_address.0,
382+
len: MMIO_LEN,
383+
irqs,
384+
}
385+
};
386+
387+
self.register_mmio_cpu_container(vm, device, &device_info)?;
388+
Ok(())
389+
}
390+
343391
/// Gets the information of the devices registered up to some point in time.
344392
pub fn get_device_info(&self) -> &HashMap<(DeviceType, String), MMIODeviceInfo> {
345393
&self.id_to_dev_info
@@ -832,4 +880,24 @@ mod tests {
832880
.allocate_mmio_resources(&mut resource_allocator, 0)
833881
.unwrap();
834882
}
883+
884+
#[test]
885+
#[cfg(target_arch = "x86_64")]
886+
fn test_register_cpu_container() {
887+
let mut device_manager = MMIODeviceManager::new();
888+
let mut resource_allocator = ResourceAllocator::new().unwrap();
889+
let mut vm = Vm::new(vec![]).unwrap();
890+
builder::setup_interrupt_controller(&mut vm).unwrap();
891+
let cpu_container = Arc::new(Mutex::new(
892+
CpuContainer::new(&mut resource_allocator, 1).unwrap(),
893+
));
894+
895+
device_manager
896+
.register_mmio_cpu_container_for_boot(vm.fd(), &mut resource_allocator, cpu_container)
897+
.unwrap();
898+
899+
device_manager
900+
.get_device(DeviceType::CpuContainer, "CpuContainer")
901+
.unwrap();
902+
}
835903
}

‎src/vmm/src/devices/acpi/cpu_container.rs‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55

66
use acpi_tables::madt::LocalAPIC;
77
use acpi_tables::{aml, Aml};
8+
use kvm_ioctls::VmFd;
89
use log::{debug, error};
910
use serde::{Deserialize, Serialize};
1011
use utils::eventfd::EventFd;
1112
use vm_memory::{GuestAddress, GuestMemoryError};
1213
use vm_superio::Trigger;
1314
use zerocopy::AsBytes;
1415

16+
use crate::device_manager::mmio::MMIO_LEN;
1517
use crate::device_manager::resources::ResourceAllocator;
1618
use crate::devices::legacy::EventFdTrigger;
1719
use crate::vmm_config::machine_config::MAX_SUPPORTED_VCPUS;
@@ -41,6 +43,8 @@ pub enum CpuContainerError {
4143
GuestMemory(#[from] GuestMemoryError),
4244
/// Failed to allocate requested resource: {0}
4345
Allocator(#[from] vm_allocator::Error),
46+
/// Failed to register file descriptor: {0}
47+
RegisterIrqFd(#[from] kvm_ioctls::Error),
4448
}
4549

4650
pub const CPU_CONTAINER_ACPI_SIZE: usize = 0xC;
@@ -78,10 +82,10 @@ impl CpuContainer {
7882
boot_count: u8,
7983
) -> Result<Self, CpuContainerError> {
8084
let gsi = resource_allocator.allocate_gsi(1)?;
81-
let mmio_address = resource_allocator.allocate_system_memory(
82-
4096,
83-
8,
84-
vm_allocator::AllocPolicy::LastMatch,
85+
let mmio_address = resource_allocator.allocate_mmio_memory(
86+
MMIO_LEN,
87+
MMIO_LEN,
88+
vm_allocator::AllocPolicy::FirstMatch,
8589
)?;
8690

8791
let mut cpu_devices = Vec::new();
@@ -103,6 +107,49 @@ impl CpuContainer {
103107
debug!("hotplug: notifying guest about new vcpus available");
104108
Ok(())
105109
}
110+
111+
pub fn bus_read(&mut self, offset: u64, data: &mut [u8]) {
112+
data.fill(0);
113+
match offset {
114+
CPU_SELECTION_OFFSET => {
115+
data[0] = self.selected_cpu;
116+
}
117+
CPU_STATUS_OFFSET => {
118+
if self.selected_cpu < MAX_SUPPORTED_VCPUS {
119+
let cpu_device = &self.cpu_devices[self.selected_cpu as usize];
120+
if cpu_device.active {
121+
data[0] |= CPU_ENABLE_BIT;
122+
}
123+
if cpu_device.inserting {
124+
data[0] |= CPU_INSERTING_BIT;
125+
}
126+
} else {
127+
error!("Out of range vCPU id: {}", self.selected_cpu)
128+
}
129+
}
130+
_ => error!("Unexpected CPU container offset"),
131+
}
132+
}
133+
134+
pub fn bus_write(&mut self, offset: u64, data: &[u8]) {
135+
match offset {
136+
CPU_SELECTION_OFFSET => self.selected_cpu = data[0],
137+
CPU_STATUS_OFFSET => {
138+
if self.selected_cpu < MAX_SUPPORTED_VCPUS {
139+
let cpu_device = &mut self.cpu_devices[self.selected_cpu as usize];
140+
if data[0] & CPU_INSERTING_BIT != 0 {
141+
cpu_device.inserting = false;
142+
}
143+
if data[0] & CPU_ENABLE_BIT != 0 {
144+
cpu_device.active = true;
145+
}
146+
} else {
147+
error!("Out of range vCPU id: {}", self.selected_cpu)
148+
}
149+
}
150+
_ => error!("Unexpected CPU container offset"),
151+
}
152+
}
106153
}
107154

108155
impl Aml for CpuContainer {

‎src/vmm/src/devices/bus.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ pub struct Bus {
5252

5353
use event_manager::{EventOps, Events, MutEventSubscriber};
5454

55+
use super::acpi::cpu_container::CpuContainer;
5556
#[cfg(target_arch = "x86_64")]
5657
use super::legacy::I8042Device;
5758
#[cfg(target_arch = "aarch64")]
@@ -69,6 +70,8 @@ pub enum BusDevice {
6970
BootTimer(Arc<Mutex<BootTimer>>),
7071
MmioTransport(Arc<Mutex<MmioTransport>>),
7172
Serial(Arc<Mutex<SerialDevice<std::io::Stdin>>>),
73+
#[cfg(target_arch = "x86_64")]
74+
CpuContainer(Arc<Mutex<CpuContainer>>),
7275
#[cfg(test)]
7376
Dummy(Arc<Mutex<DummyDevice>>),
7477
#[cfg(test)]
@@ -134,6 +137,8 @@ impl BusDevice {
134137
Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
135138
Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
136139
Self::Serial(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
140+
#[cfg(target_arch = "x86_64")]
141+
Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
137142
#[cfg(test)]
138143
Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
139144
#[cfg(test)]
@@ -150,6 +155,8 @@ impl BusDevice {
150155
Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
151156
Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
152157
Self::Serial(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
158+
#[cfg(target_arch = "x86_64")]
159+
Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
153160
#[cfg(test)]
154161
Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
155162
#[cfg(test)]

0 commit comments

Comments
 (0)