Skip to content

Commit 26ebb29

Browse files
JamesC1305JamesC1305
committed
Add vCPU hotplug controller and extend GED
Define new ACPI devices related to hotplugging, including a hotplug controller, a CPU container, and individual CPU devices. Add hotplugging devices to GED. Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>
1 parent 4aa14de commit 26ebb29

File tree

6 files changed

+521
-41
lines changed

6 files changed

+521
-41
lines changed

‎src/vmm/src/builder.rs‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ use vm_superio::Serial;
2929

3030
#[cfg(target_arch = "x86_64")]
3131
use crate::acpi;
32+
#[cfg(target_arch = "x86_64")]
33+
use crate::arch::DeviceType;
3234
use crate::arch::InitrdConfig;
3335
#[cfg(target_arch = "aarch64")]
3436
use crate::construct_kvm_mpidrs;
@@ -48,6 +50,8 @@ use crate::device_manager::persist::{
4850
};
4951
use crate::device_manager::resources::ResourceAllocator;
5052
#[cfg(target_arch = "x86_64")]
53+
use crate::devices::acpi::cpu_container::{CpuContainer, CpuContainerError};
54+
#[cfg(target_arch = "x86_64")]
5155
use crate::devices::acpi::vmgenid::{VmGenId, VmGenIdError};
5256
use crate::devices::legacy::serial::SerialOut;
5357
#[cfg(target_arch = "aarch64")]
@@ -96,6 +100,9 @@ pub enum StartMicrovmError {
96100
/// Error creating VMGenID device: {0}
97101
#[cfg(target_arch = "x86_64")]
98102
CreateVMGenID(VmGenIdError),
103+
/// Error creating CpuContainer device: {0}
104+
#[cfg(target_arch = "x86_64")]
105+
CreateCpuContainer(#[from] CpuContainerError),
99106
/// Invalid Memory Configuration: {0}
100107
GuestMemory(crate::vstate::memory::MemoryError),
101108
/// Cannot load initrd due to an invalid memory configuration.
@@ -172,10 +179,10 @@ fn create_vmm_and_vcpus(
172179
.map_err(VmmError::EventFd)
173180
.map_err(Internal)?;
174181

175-
let resource_allocator = ResourceAllocator::new()?;
182+
let mut resource_allocator = ResourceAllocator::new()?;
176183

177184
// Instantiate the MMIO device manager.
178-
let mmio_device_manager = MMIODeviceManager::new();
185+
let mut mmio_device_manager = MMIODeviceManager::new();
179186

180187
// Instantiate ACPI device manager.
181188
#[cfg(target_arch = "x86_64")]
@@ -326,6 +333,9 @@ pub fn build_microvm_for_boot(
326333
attach_boot_timer_device(&mut vmm, request_ts)?;
327334
}
328335

336+
#[cfg(target_arch = "x86_64")]
337+
attach_cpu_container_device(&mut vmm, vm_resources.vm_config.vcpu_count)?;
338+
329339
if let Some(balloon) = vm_resources.balloon.get() {
330340
attach_balloon_device(&mut vmm, &mut boot_cmdline, balloon, event_manager)?;
331341
}
@@ -1024,6 +1034,16 @@ fn attach_balloon_device(
10241034
attach_virtio_device(event_manager, vmm, id, balloon.clone(), cmdline, false)
10251035
}
10261036

1037+
fn attach_cpu_container_device(vmm: &mut Vmm, vcpu_count: u8) -> Result<(), StartMicrovmError> {
1038+
let cpu_container = Arc::new(Mutex::new(CpuContainer::new(
1039+
&mut vmm.resource_allocator,
1040+
vcpu_count,
1041+
)?));
1042+
vmm.acpi_device_manager
1043+
.attach_cpu_container(cpu_container.clone(), vmm.vm.fd());
1044+
Ok(())
1045+
}
1046+
10271047
// Adds `O_NONBLOCK` to the stdout flags.
10281048
pub(crate) fn set_stdout_nonblocking() {
10291049
// SAFETY: Call is safe since parameters are valid.
@@ -1048,6 +1068,8 @@ pub mod tests {
10481068
use super::*;
10491069
use crate::arch::DeviceType;
10501070
use crate::device_manager::resources::ResourceAllocator;
1071+
#[cfg(target_arch = "x86_64")]
1072+
use crate::devices::acpi::cpu_container::CpuContainer;
10511073
use crate::devices::virtio::block::CacheType;
10521074
use crate::devices::virtio::rng::device::ENTROPY_DEV_ID;
10531075
use crate::devices::virtio::vsock::{TYPE_VSOCK, VSOCK_DEV_ID};
@@ -1126,9 +1148,17 @@ pub mod tests {
11261148

11271149
let mut vm = Vm::new(vec![]).unwrap();
11281150
vm.memory_init(&guest_memory, false).unwrap();
1151+
1152+
#[cfg(target_arch = "x86_64")]
1153+
let mut resource_allocator = ResourceAllocator::new().unwrap();
1154+
#[cfg(target_arch = "aarch64")]
1155+
let resource_allocator = ResourceAllocator::new().unwrap();
1156+
11291157
let mmio_device_manager = MMIODeviceManager::new();
1158+
11301159
#[cfg(target_arch = "x86_64")]
11311160
let acpi_device_manager = ACPIDeviceManager::new();
1161+
11321162
#[cfg(target_arch = "x86_64")]
11331163
let pio_device_manager = PortIODeviceManager::new(
11341164
Arc::new(Mutex::new(SerialWrapper {
@@ -1148,6 +1178,14 @@ pub mod tests {
11481178
#[cfg(target_arch = "x86_64")]
11491179
setup_interrupt_controller(&mut vm).unwrap();
11501180

1181+
#[cfg(target_arch = "x86_64")]
1182+
{
1183+
let cpu_container = Arc::new(Mutex::new(
1184+
CpuContainer::new(&mut resource_allocator, 1).unwrap(),
1185+
));
1186+
acpi_device_manager.attach_cpu_container(cpu_container.clone, vm.fd())
1187+
}
1188+
11511189
#[cfg(target_arch = "aarch64")]
11521190
{
11531191
let exit_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
@@ -1166,7 +1204,7 @@ pub mod tests {
11661204
vcpus_exit_evt,
11671205
#[cfg(target_arch = "x86_64")]
11681206
seccomp_filters: crate::seccomp_filters::get_empty_filters(),
1169-
resource_allocator: ResourceAllocator::new().unwrap(),
1207+
resource_allocator,
11701208
mmio_device_manager,
11711209
#[cfg(target_arch = "x86_64")]
11721210
pio_device_manager,

‎src/vmm/src/device_manager/acpi.rs‎

Lines changed: 152 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,28 @@
11
// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
// SPDX-License-Identifier: Apache-2.0
33

4+
use std::sync::{Arc, Mutex};
5+
46
use acpi_tables::{aml, Aml};
57
use kvm_ioctls::VmFd;
68

9+
use crate::devices::acpi::cpu_container::CpuContainer;
710
use crate::devices::acpi::vmgenid::VmGenId;
811

912
#[derive(Debug)]
1013
pub struct ACPIDeviceManager {
1114
/// VMGenID device
1215
pub vmgenid: Option<VmGenId>,
16+
pub cpu_container: Option<Arc<Mutex<CpuContainer>>>,
1317
}
1418

1519
impl ACPIDeviceManager {
1620
/// Create a new ACPIDeviceManager object
1721
pub fn new() -> Self {
18-
Self { vmgenid: None }
22+
Self {
23+
vmgenid: None,
24+
cpu_container: None,
25+
}
1926
}
2027

2128
/// Attach a new VMGenID device to the microVM
@@ -31,6 +38,24 @@ impl ACPIDeviceManager {
3138
Ok(())
3239
}
3340

41+
pub fn attach_cpu_container(
42+
&mut self,
43+
cpu_container: Arc<Mutex<CpuContainer>>,
44+
vm_fd: &VmFd,
45+
) -> Result<(), kvm_ioctls::Error> {
46+
let locked_container = cpu_container.lock().expect("Poisoned lock");
47+
vm_fd.register_irqfd(&locked_container.acpi_interrupt_evt, locked_container.gsi)?;
48+
self.cpu_container = Some(cpu_container);
49+
Ok(())
50+
}
51+
52+
pub fn notify_cpu_container(&mut self) -> Result<(), std::io::Error> {
53+
if let Some(container) = &mut self.cpu_container {
54+
container.lock().expect("Poisoned lock").notify_guest()?;
55+
}
56+
Ok(())
57+
}
58+
3459
/// If it exists, notify guest VMGenID device that we have resumed from a snapshot.
3560
pub fn notify_vmgenid(&mut self) -> Result<(), std::io::Error> {
3661
if let Some(vmgenid) = &mut self.vmgenid {
@@ -42,44 +67,134 @@ impl ACPIDeviceManager {
4267

4368
impl Aml for ACPIDeviceManager {
4469
fn append_aml_bytes(&self, v: &mut Vec<u8>) {
45-
// If we have a VMGenID device, create the AML for the device and GED interrupt handler
46-
self.vmgenid.as_ref().inspect(|vmgenid| {
47-
// AML for GED
48-
aml::Device::new(
49-
"_SB_.GED_".into(),
50-
vec![
51-
&aml::Name::new("_HID".into(), &"ACPI0013"),
52-
&aml::Name::new(
53-
"_CRS".into(),
54-
&aml::ResourceTemplate::new(vec![&aml::Interrupt::new(
70+
// Depending on what devices are available, generate AML for GED.
71+
match (self.cpu_container.as_ref(), self.vmgenid.as_ref()) {
72+
(Some(container), Some(vmgenid)) => {
73+
let locked_container = container.lock().expect("Poisoned lock");
74+
aml::Device::new(
75+
"_SB_.GED_".into(),
76+
vec![
77+
&aml::Name::new("_HID".into(), &"ACPI0013"),
78+
&aml::Name::new(
79+
"_CRS".into(),
80+
&aml::ResourceTemplate::new(vec![
81+
&aml::Interrupt::new(
82+
true,
83+
true,
84+
false,
85+
false,
86+
locked_container.gsi,
87+
),
88+
&aml::Interrupt::new(true, true, false, false, vmgenid.gsi),
89+
]),
90+
),
91+
&aml::Method::new(
92+
"_EVT".into(),
93+
1,
5594
true,
95+
vec![
96+
&aml::If::new(
97+
#[allow(clippy::cast_possible_truncation)]
98+
&aml::Equal::new(&aml::Arg(0), &(locked_container.gsi as u8)),
99+
vec![&aml::MethodCall::new(
100+
aml::Path::new("\\_SB_.CPUS.CSCN"),
101+
vec![],
102+
)],
103+
),
104+
&aml::If::new(
105+
#[allow(clippy::cast_possible_truncation)]
106+
&aml::Equal::new(&aml::Arg(0), &(vmgenid.gsi as u8)),
107+
vec![&aml::Notify::new(
108+
&aml::Path::new("\\_SB_.VGEN"),
109+
&0x80usize,
110+
)],
111+
),
112+
],
113+
),
114+
],
115+
)
116+
.append_aml_bytes(v);
117+
locked_container.append_aml_bytes(v);
118+
vmgenid.append_aml_bytes(v);
119+
}
120+
(Some(container), None) => {
121+
let locked_container = container.lock().expect("Poisoned lock");
122+
aml::Device::new(
123+
"_SB_.GED_".into(),
124+
vec![
125+
&aml::Name::new("_HID".into(), &"ACPI0013"),
126+
&aml::Name::new(
127+
"_CRS".into(),
128+
&aml::ResourceTemplate::new(vec![&aml::Interrupt::new(
129+
true,
130+
true,
131+
false,
132+
false,
133+
locked_container.gsi,
134+
)]),
135+
),
136+
&aml::Method::new(
137+
"_EVT".into(),
138+
1,
56139
true,
57-
false,
58-
false,
59-
vmgenid.gsi,
60-
)]),
61-
),
62-
&aml::Method::new(
63-
"_EVT".into(),
64-
1,
65-
true,
66-
vec![&aml::If::new(
67-
// We know that the maximum IRQ number fits in a u8. We have up to 32
68-
// IRQs in x86 and up to 128 in ARM (look into
69-
// `vmm::crate::arch::layout::IRQ_MAX`)
70-
#[allow(clippy::cast_possible_truncation)]
71-
&aml::Equal::new(&aml::Arg(0), &(vmgenid.gsi as u8)),
72-
vec![&aml::Notify::new(
73-
&aml::Path::new("\\_SB_.VGEN"),
74-
&0x80usize,
140+
vec![&aml::If::new(
141+
#[allow(clippy::cast_possible_truncation)]
142+
&aml::Equal::new(&aml::Arg(0), &(locked_container.gsi as u8)),
143+
vec![&aml::MethodCall::new(
144+
aml::Path::new("\\_SB_.CPUS.CSCN"),
145+
vec![],
146+
)],
75147
)],
76-
)],
77-
),
78-
],
79-
)
80-
.append_aml_bytes(v);
81-
// AML for VMGenID itself.
82-
vmgenid.append_aml_bytes(v);
83-
});
148+
),
149+
],
150+
)
151+
.append_aml_bytes(v);
152+
locked_container.append_aml_bytes(v);
153+
}
154+
(None, Some(vmgenid)) => {
155+
aml::Device::new(
156+
"_SB_.GED_".into(),
157+
vec![
158+
&aml::Name::new("_HID".into(), &"ACPI0013"),
159+
&aml::Name::new(
160+
"_CRS".into(),
161+
&aml::ResourceTemplate::new(vec![&aml::Interrupt::new(
162+
true,
163+
true,
164+
false,
165+
false,
166+
vmgenid.gsi,
167+
)]),
168+
),
169+
&aml::Method::new(
170+
"_EVT".into(),
171+
1,
172+
true,
173+
vec![&aml::If::new(
174+
#[allow(clippy::cast_possible_truncation)]
175+
&aml::Equal::new(&aml::Arg(0), &(vmgenid.gsi as u8)),
176+
vec![&aml::Notify::new(
177+
&aml::Path::new("\\_SB_.VGEN"),
178+
&0x80usize,
179+
)],
180+
)],
181+
),
182+
],
183+
)
184+
.append_aml_bytes(v);
185+
vmgenid.append_aml_bytes(v);
186+
}
187+
(None, None) => {
188+
aml::Device::new(
189+
"_SB_.GED_".into(),
190+
vec![
191+
&aml::Name::new("_HID".into(), &"ACPI0013"),
192+
&aml::Name::new("_CRS".into(), &aml::ResourceTemplate::new(vec![])),
193+
&aml::Method::new("_EVT".into(), 1, true, vec![]),
194+
],
195+
)
196+
.append_aml_bytes(v);
197+
}
198+
}
84199
}
85200
}

‎src/vmm/src/device_manager/persist.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ use super::resources::ResourceAllocator;
1919
#[cfg(target_arch = "aarch64")]
2020
use crate::arch::DeviceType;
2121
#[cfg(target_arch = "x86_64")]
22+
use crate::devices::acpi::cpu_container::CpuContainer;
23+
#[cfg(target_arch = "x86_64")]
2224
use crate::devices::acpi::vmgenid::{VMGenIDState, VMGenIdConstructorArgs, VmGenId, VmGenIdError};
2325
use crate::devices::virtio::balloon::persist::{BalloonConstructorArgs, BalloonState};
2426
use crate::devices::virtio::balloon::{Balloon, BalloonError};
@@ -241,6 +243,7 @@ pub struct ACPIDeviceManagerConstructorArgs<'a> {
241243
pub mem: &'a GuestMemoryMmap,
242244
pub resource_allocator: &'a mut ResourceAllocator,
243245
pub vm: &'a VmFd,
246+
pub cpu_container: Arc<Mutex<CpuContainer>>,
244247
}
245248

246249
#[cfg(target_arch = "x86_64")]

0 commit comments

Comments
 (0)