Skip to content

Commit 7d2af95

Browse files
JamesC1305JamesC1305
committed
MMIO Portion of hotplugging
Implement MMIO for the CpuContainer. This allows the AML methods to read from and write to the defined AML fields. Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>
1 parent 26ebb29 commit 7d2af95

File tree

8 files changed

+200
-33
lines changed

8 files changed

+200
-33
lines changed

‎src/vmm/src/arch/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ pub enum DeviceType {
4141
Rtc,
4242
/// Device Type: BootTimer.
4343
BootTimer,
44+
/// Device Type: CpuContainer
45+
#[cfg(target_arch = "x86_64")]
46+
CpuContainer,
4447
}
4548

4649
/// Type for passing information about the initrd in the guest memory.

‎src/vmm/src/builder.rs‎

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ use crate::cpu_config::templates::{
3838
CpuConfiguration, CustomCpuTemplate, GetCpuTemplate, GetCpuTemplateError, GuestConfigError,
3939
KvmCapability,
4040
};
41+
use crate::cpu_config::x86_64::cpuid::{self, Cpuid};
4142
#[cfg(target_arch = "x86_64")]
4243
use crate::device_manager::acpi::ACPIDeviceManager;
4344
#[cfg(target_arch = "x86_64")]
@@ -86,6 +87,9 @@ pub enum StartMicrovmError {
8687
/// Unable to attach the VMGenID device: {0}
8788
#[cfg(target_arch = "x86_64")]
8889
AttachVmgenidDevice(kvm_ioctls::Error),
90+
/// Unable to attach the CpuContainer device: {0}
91+
#[cfg(target_arch = "x86_64")]
92+
AttachCpuContainerDevice(kvm_ioctls::Error),
8993
/// System configuration error: {0}
9094
ConfigureSystem(crate::arch::ConfigurationError),
9195
/// Failed to create guest config: {0}
@@ -179,10 +183,10 @@ fn create_vmm_and_vcpus(
179183
.map_err(VmmError::EventFd)
180184
.map_err(Internal)?;
181185

182-
let mut resource_allocator = ResourceAllocator::new()?;
186+
let resource_allocator = ResourceAllocator::new()?;
183187

184188
// Instantiate the MMIO device manager.
185-
let mut mmio_device_manager = MMIODeviceManager::new();
189+
let mmio_device_manager = MMIODeviceManager::new();
186190

187191
// Instantiate ACPI device manager.
188192
#[cfg(target_arch = "x86_64")]
@@ -553,14 +557,20 @@ pub fn build_microvm_from_snapshot(
553557

554558
#[cfg(target_arch = "x86_64")]
555559
{
556-
let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
557-
mem: &guest_memory,
558-
resource_allocator: &mut vmm.resource_allocator,
559-
vm: vmm.vm.fd(),
560-
};
560+
if let Some(BusDevice::CpuContainer(container)) =
561+
vmm.get_bus_device(DeviceType::CpuContainer, "CpuContainer")
562+
{
563+
let container_ref = container.clone();
564+
let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
565+
mem: &guest_memory,
566+
resource_allocator: &mut vmm.resource_allocator,
567+
vm: vmm.vm.fd(),
568+
cpu_container: container_ref,
569+
};
561570

562-
vmm.acpi_device_manager =
563-
ACPIDeviceManager::restore(acpi_ctor_args, &microvm_state.acpi_dev_state)?;
571+
vmm.acpi_device_manager =
572+
ACPIDeviceManager::restore(acpi_ctor_args, &microvm_state.acpi_dev_state)?;
573+
}
564574

565575
// Inject the notification to VMGenID that we have resumed from a snapshot.
566576
// This needs to happen before we resume vCPUs, so that we minimize the time between vCPUs
@@ -1034,13 +1044,23 @@ fn attach_balloon_device(
10341044
attach_virtio_device(event_manager, vmm, id, balloon.clone(), cmdline, false)
10351045
}
10361046

1047+
#[cfg(target_arch = "x86_64")]
10371048
fn attach_cpu_container_device(vmm: &mut Vmm, vcpu_count: u8) -> Result<(), StartMicrovmError> {
10381049
let cpu_container = Arc::new(Mutex::new(CpuContainer::new(
10391050
&mut vmm.resource_allocator,
10401051
vcpu_count,
10411052
)?));
10421053
vmm.acpi_device_manager
1043-
.attach_cpu_container(cpu_container.clone(), vmm.vm.fd());
1054+
.attach_cpu_container(cpu_container.clone(), vmm.vm.fd())
1055+
.map_err(StartMicrovmError::AttachCpuContainerDevice)?;
1056+
vmm.mmio_device_manager
1057+
.register_mmio_cpu_container_for_boot(
1058+
vmm.vm.fd(),
1059+
&mut vmm.resource_allocator,
1060+
cpu_container,
1061+
)
1062+
.map_err(StartMicrovmError::RegisterMmioDevice)?;
1063+
10441064
Ok(())
10451065
}
10461066

@@ -1154,10 +1174,13 @@ pub mod tests {
11541174
#[cfg(target_arch = "aarch64")]
11551175
let resource_allocator = ResourceAllocator::new().unwrap();
11561176

1177+
#[cfg(target_arch = "x86_64")]
1178+
let mut mmio_device_manager = MMIODeviceManager::new();
1179+
#[cfg(target_arch = "aarch64")]
11571180
let mmio_device_manager = MMIODeviceManager::new();
11581181

11591182
#[cfg(target_arch = "x86_64")]
1160-
let acpi_device_manager = ACPIDeviceManager::new();
1183+
let mut acpi_device_manager = ACPIDeviceManager::new();
11611184

11621185
#[cfg(target_arch = "x86_64")]
11631186
let pio_device_manager = PortIODeviceManager::new(
@@ -1175,15 +1198,22 @@ pub mod tests {
11751198
)
11761199
.unwrap();
11771200

1178-
#[cfg(target_arch = "x86_64")]
1179-
setup_interrupt_controller(&mut vm).unwrap();
1180-
11811201
#[cfg(target_arch = "x86_64")]
11821202
{
1203+
setup_interrupt_controller(&mut vm).unwrap();
11831204
let cpu_container = Arc::new(Mutex::new(
11841205
CpuContainer::new(&mut resource_allocator, 1).unwrap(),
11851206
));
1186-
acpi_device_manager.attach_cpu_container(cpu_container.clone, vm.fd())
1207+
acpi_device_manager
1208+
.attach_cpu_container(cpu_container.clone(), vm.fd())
1209+
.unwrap();
1210+
mmio_device_manager
1211+
.register_mmio_cpu_container_for_boot(
1212+
vm.fd(),
1213+
&mut resource_allocator,
1214+
cpu_container,
1215+
)
1216+
.unwrap();
11871217
}
11881218

11891219
#[cfg(target_arch = "aarch64")]

‎src/vmm/src/device_manager/acpi.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ impl ACPIDeviceManager {
4343
cpu_container: Arc<Mutex<CpuContainer>>,
4444
vm_fd: &VmFd,
4545
) -> Result<(), kvm_ioctls::Error> {
46-
let locked_container = cpu_container.lock().expect("Poisoned lock");
47-
vm_fd.register_irqfd(&locked_container.acpi_interrupt_evt, locked_container.gsi)?;
46+
{
47+
let locked_container = cpu_container.lock().expect("Poisoned lock");
48+
vm_fd.register_irqfd(&locked_container.acpi_interrupt_evt, locked_container.gsi)?;
49+
}
4850
self.cpu_container = Some(cpu_container);
4951
Ok(())
5052
}

‎src/vmm/src/device_manager/mmio.rs‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ use super::resources::ResourceAllocator;
2424
use crate::arch::aarch64::DeviceInfoForFDT;
2525
use crate::arch::DeviceType;
2626
use crate::arch::DeviceType::Virtio;
27+
#[cfg(target_arch = "x86_64")]
28+
use crate::devices::acpi::cpu_container::CpuContainer;
2729
#[cfg(target_arch = "aarch64")]
2830
use crate::devices::legacy::{RTCDevice, SerialDevice};
2931
use crate::devices::pseudo::BootTimer;
@@ -319,7 +321,7 @@ impl MMIODeviceManager {
319321
self.register_mmio_device(
320322
identifier,
321323
device_info,
322-
Arc::new(Mutex::new(BusDevice::RTCDevice(rtc))),
324+
BusDevice::RTCDevice(Arc::new(Mutex::new(rtc))),
323325
)
324326
}
325327

@@ -340,6 +342,52 @@ impl MMIODeviceManager {
340342
)
341343
}
342344

345+
#[cfg(target_arch = "x86_64")]
346+
pub fn register_mmio_cpu_container(
347+
&mut self,
348+
vm: &VmFd,
349+
device: Arc<Mutex<CpuContainer>>,
350+
device_info: &MMIODeviceInfo,
351+
) -> Result<(), MmioError> {
352+
let identifier = (
353+
DeviceType::CpuContainer,
354+
DeviceType::CpuContainer.to_string(),
355+
);
356+
{
357+
let container = device.lock().expect("Poisoned lock");
358+
vm.register_irqfd(&container.mmio_interrupt_evt, device_info.irqs[0])
359+
.map_err(MmioError::RegisterIrqFd)?;
360+
}
361+
362+
self.register_mmio_device(
363+
identifier,
364+
device_info.clone(),
365+
BusDevice::CpuContainer(device),
366+
)?;
367+
Ok(())
368+
}
369+
370+
#[cfg(target_arch = "x86_64")]
371+
pub fn register_mmio_cpu_container_for_boot(
372+
&mut self,
373+
vm: &VmFd,
374+
resource_allocator: &mut ResourceAllocator,
375+
device: Arc<Mutex<CpuContainer>>,
376+
) -> Result<(), MmioError> {
377+
let device_info = {
378+
let locked_container = device.lock().expect("Poisoned lock");
379+
let irqs = resource_allocator.allocate_gsi(1)?;
380+
MMIODeviceInfo {
381+
addr: locked_container.mmio_address.0,
382+
len: MMIO_LEN,
383+
irqs,
384+
}
385+
};
386+
387+
self.register_mmio_cpu_container(vm, device, &device_info)?;
388+
Ok(())
389+
}
390+
343391
/// Gets the information of the devices registered up to some point in time.
344392
pub fn get_device_info(&self) -> &HashMap<(DeviceType, String), MMIODeviceInfo> {
345393
&self.id_to_dev_info
@@ -832,4 +880,24 @@ mod tests {
832880
.allocate_mmio_resources(&mut resource_allocator, 0)
833881
.unwrap();
834882
}
883+
884+
#[test]
885+
#[cfg(target_arch = "x86_64")]
886+
fn test_register_cpu_container() {
887+
let mut device_manager = MMIODeviceManager::new();
888+
let mut resource_allocator = ResourceAllocator::new().unwrap();
889+
let mut vm = Vm::new(vec![]).unwrap();
890+
builder::setup_interrupt_controller(&mut vm).unwrap();
891+
let cpu_container = Arc::new(Mutex::new(
892+
CpuContainer::new(&mut resource_allocator, 1).unwrap(),
893+
));
894+
895+
device_manager
896+
.register_mmio_cpu_container_for_boot(vm.fd(), &mut resource_allocator, cpu_container)
897+
.unwrap();
898+
899+
device_manager
900+
.get_device(DeviceType::CpuContainer, "CpuContainer")
901+
.unwrap();
902+
}
835903
}

‎src/vmm/src/devices/acpi/cpu_container.rs‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55

66
use acpi_tables::madt::LocalAPIC;
77
use acpi_tables::{aml, Aml};
8+
use kvm_ioctls::VmFd;
89
use log::{debug, error};
910
use serde::{Deserialize, Serialize};
1011
use utils::eventfd::EventFd;
1112
use vm_memory::{GuestAddress, GuestMemoryError};
1213
use vm_superio::Trigger;
1314
use zerocopy::AsBytes;
1415

16+
use crate::device_manager::mmio::MMIO_LEN;
1517
use crate::device_manager::resources::ResourceAllocator;
1618
use crate::devices::legacy::EventFdTrigger;
1719
use crate::vmm_config::machine_config::MAX_SUPPORTED_VCPUS;
@@ -41,6 +43,8 @@ pub enum CpuContainerError {
4143
GuestMemory(#[from] GuestMemoryError),
4244
/// Failed to allocate requested resource: {0}
4345
Allocator(#[from] vm_allocator::Error),
46+
/// Failed to register file descriptor: {0}
47+
RegisterIrqFd(#[from] kvm_ioctls::Error),
4448
}
4549

4650
pub const CPU_CONTAINER_ACPI_SIZE: usize = 0xC;
@@ -78,10 +82,10 @@ impl CpuContainer {
7882
boot_count: u8,
7983
) -> Result<Self, CpuContainerError> {
8084
let gsi = resource_allocator.allocate_gsi(1)?;
81-
let mmio_address = resource_allocator.allocate_system_memory(
82-
4096,
83-
8,
84-
vm_allocator::AllocPolicy::LastMatch,
85+
let mmio_address = resource_allocator.allocate_mmio_memory(
86+
MMIO_LEN,
87+
MMIO_LEN,
88+
vm_allocator::AllocPolicy::FirstMatch,
8589
)?;
8690

8791
let mut cpu_devices = Vec::new();
@@ -103,6 +107,49 @@ impl CpuContainer {
103107
debug!("hotplug: notifying guest about new vcpus available");
104108
Ok(())
105109
}
110+
111+
pub fn bus_read(&mut self, offset: u64, data: &mut [u8]) {
112+
data.fill(0);
113+
match offset {
114+
CPU_SELECTION_OFFSET => {
115+
data[0] = self.selected_cpu;
116+
}
117+
CPU_STATUS_OFFSET => {
118+
if self.selected_cpu < MAX_SUPPORTED_VCPUS {
119+
let cpu_device = &self.cpu_devices[self.selected_cpu as usize];
120+
if cpu_device.active {
121+
data[0] |= CPU_ENABLE_BIT;
122+
}
123+
if cpu_device.inserting {
124+
data[0] |= CPU_INSERTING_BIT;
125+
}
126+
} else {
127+
error!("Out of range vCPU id: {}", self.selected_cpu)
128+
}
129+
}
130+
_ => error!("Unexpected CPU container offset"),
131+
}
132+
}
133+
134+
pub fn bus_write(&mut self, offset: u64, data: &[u8]) {
135+
match offset {
136+
CPU_SELECTION_OFFSET => self.selected_cpu = data[0],
137+
CPU_STATUS_OFFSET => {
138+
if self.selected_cpu < MAX_SUPPORTED_VCPUS {
139+
let cpu_device = &mut self.cpu_devices[self.selected_cpu as usize];
140+
if data[0] & CPU_INSERTING_BIT != 0 {
141+
cpu_device.inserting = false;
142+
}
143+
if data[0] & CPU_ENABLE_BIT != 0 {
144+
cpu_device.active = true;
145+
}
146+
} else {
147+
error!("Out of range vCPU id: {}", self.selected_cpu)
148+
}
149+
}
150+
_ => error!("Unexpected CPU container offset"),
151+
}
152+
}
106153
}
107154

108155
impl Aml for CpuContainer {

‎src/vmm/src/devices/bus.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ pub struct Bus {
5252

5353
use event_manager::{EventOps, Events, MutEventSubscriber};
5454

55+
#[cfg(target_arch = "x86_64")]
56+
use super::acpi::cpu_container::CpuContainer;
5557
#[cfg(target_arch = "x86_64")]
5658
use super::legacy::I8042Device;
5759
#[cfg(target_arch = "aarch64")]
@@ -69,6 +71,8 @@ pub enum BusDevice {
6971
BootTimer(Arc<Mutex<BootTimer>>),
7072
MmioTransport(Arc<Mutex<MmioTransport>>),
7173
Serial(Arc<Mutex<SerialDevice<std::io::Stdin>>>),
74+
#[cfg(target_arch = "x86_64")]
75+
CpuContainer(Arc<Mutex<CpuContainer>>),
7276
#[cfg(test)]
7377
Dummy(Arc<Mutex<DummyDevice>>),
7478
#[cfg(test)]
@@ -134,6 +138,8 @@ impl BusDevice {
134138
Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
135139
Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
136140
Self::Serial(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
141+
#[cfg(target_arch = "x86_64")]
142+
Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
137143
#[cfg(test)]
138144
Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_read(offset, data),
139145
#[cfg(test)]
@@ -150,6 +156,8 @@ impl BusDevice {
150156
Self::BootTimer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
151157
Self::MmioTransport(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
152158
Self::Serial(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
159+
#[cfg(target_arch = "x86_64")]
160+
Self::CpuContainer(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
153161
#[cfg(test)]
154162
Self::Dummy(x) => x.lock().expect("Poisoned lock").bus_write(offset, data),
155163
#[cfg(test)]

0 commit comments

Comments
 (0)