Merge pull request #93 from mythril-hypervisor/basic-guest-apic-support

Add support for guest booting with APIC enabled

Author: dlrobertson

mythril/src/emulate/cpuid.rs

@@ -20,6 +20,9 @@ pub fn emulate_cpuid(
 
         // Hide hypervisor feature
         res.ecx &= !(1 << 31);
+
+        // Hide TSC deadline timer
+        res.ecx &= !(1 << 24);
     }
 
     guest_cpu.rax = res.eax as u64 | (guest_cpu.rax & 0xffffffff00000000);

mythril/src/kmain.rs

@@ -63,7 +63,7 @@ fn build_vm(
     );
 
     let mut madt = acpi::madt::MADTBuilder::<[_; 8]>::new();
-    madt.set_ica(0xfee00000);
+    madt.set_ica(vm::GUEST_LOCAL_APIC_ADDR.as_u64() as u32);
     madt.add_ics(acpi::madt::Ics::LocalApic {
         apic_id: 0,
         apic_uid: 0,
@@ -117,10 +117,8 @@ fn build_vm(
     device_map
         .register_device(virtdev::rtc::CmosRtc::new(cfg.memory))
         .unwrap();
-
-    //TODO: this should actually be per-vcpu
     device_map
-        .register_device(virtdev::lapic::LocalApic::new())
+        .register_device(virtdev::ioapic::IoApic::new())
         .unwrap();
 
     let mut fw_cfg_builder = virtdev::qemu_fw_cfg::QemuFwCfgBuilder::new();

mythril/src/memory.rs

@@ -15,6 +15,12 @@ use x86::controlregs::Cr0;
 
 #[repr(align(4096))]
 pub struct Raw4kPage(pub [u8; BASE_PAGE_SIZE]);
+impl Raw4kPage {
+    pub fn as_ptr(&self) -> *const u8 {
+        self.0.as_ptr()
+    }
+}
+
 impl Default for Raw4kPage {
     fn default() -> Self {
         Raw4kPage([0u8; BASE_PAGE_SIZE])
@@ -148,7 +154,7 @@ impl Add<usize> for Guest4LevelPagingAddr {
 pub struct GuestPhysAddr(u64);
 
 impl GuestPhysAddr {
-    pub fn new(addr: u64) -> Self {
+    pub const fn new(addr: u64) -> Self {
         Self(addr)
     }
 

mythril/src/vcpu.rs

@@ -65,6 +65,7 @@ pub enum InjectedInterruptType {
 pub struct VCpu {
     pub vm: Arc<RwLock<VirtualMachine>>,
     pub vmcs: vmcs::ActiveVmcs,
+    _local_apic: virtdev::lapic::LocalApic,
     pending_interrupts: BTreeMap<u8, InjectedInterruptType>,
     stack: Vec<u8>,
 }
@@ -85,15 +86,22 @@ impl VCpu {
         let mut vcpu = Box::pin(Self {
             vm: vm,
             vmcs: vmcs,
+            _local_apic: virtdev::lapic::LocalApic::new(),
             stack: stack,
             pending_interrupts: BTreeMap::new(),
         });
 
-        // All VCpus in a VM must share the same address space (except for the
-        // local apic)
+        // All VCpus in a VM must share the same address space
         let eptp = vcpu.vm.read().guest_space.eptp();
         vcpu.vmcs.write_field(vmcs::VmcsField::EptPointer, eptp)?;
 
+        // Setup access for our local apic
+        let apic_access_addr = vcpu.vm.read().apic_access_page.as_ptr() as u64;
+        vcpu.vmcs
+            .write_field(vmcs::VmcsField::ApicAccessAddr, apic_access_addr)?;
+
+        //TODO: set a per-core virtual apic page
+
         let stack_base = vcpu.stack.as_ptr() as u64 + vcpu.stack.len() as u64
             - mem::size_of::<*const Self>() as u64;
 
@@ -272,6 +280,7 @@ impl VCpu {
         vmcs.write_with_fixed(
             vmcs::VmcsField::CpuBasedVmExecControl,
             (vmcs::CpuBasedCtrlFlags::UNCOND_IO_EXITING
+                | vmcs::CpuBasedCtrlFlags::TPR_SHADOW
                 | vmcs::CpuBasedCtrlFlags::ACTIVATE_MSR_BITMAP
                 | vmcs::CpuBasedCtrlFlags::ACTIVATE_SECONDARY_CONTROLS)
                 .bits(),
@@ -310,7 +319,8 @@ impl VCpu {
         vmcs.write_with_fixed(
             vmcs::VmcsField::VmExitControls,
             (vmcs::VmExitCtrlFlags::IA32E_MODE
-                | vmcs::VmExitCtrlFlags::ACK_INTR_ON_EXIT)
+                | vmcs::VmExitCtrlFlags::ACK_INTR_ON_EXIT
+                | vmcs::VmExitCtrlFlags::SAVE_GUEST_EFER)
                 .bits(),
             msr::IA32_VMX_EXIT_CTLS,
         )?;
@@ -501,6 +511,9 @@ impl VCpu {
                 }
                 self.skip_emulated_instruction()?;
             }
+            vmexit::ExitInformation::ApicAccess(_info) => {
+                self.skip_emulated_instruction()?;
+            }
             vmexit::ExitInformation::CrAccess(info) => {
                 emulate::controlreg::emulate_access(self, guest_cpu, info)?;
                 self.skip_emulated_instruction()?;

mythril/src/virtdev/ioapic.rs

@@ -0,0 +1,33 @@
+use crate::error::Result;
+use crate::memory::GuestPhysAddr;
+use crate::virtdev::{DeviceRegion, EmulatedDevice, Event};
+use alloc::sync::Arc;
+use alloc::vec::Vec;
+use spin::RwLock;
+
+#[derive(Default)]
+pub struct IoApic;
+
+impl IoApic {
+    pub fn new() -> Arc<RwLock<Self>> {
+        Arc::new(RwLock::new(IoApic {}))
+    }
+}
+
+impl EmulatedDevice for IoApic {
+    fn services(&self) -> Vec<DeviceRegion> {
+        vec![
+            DeviceRegion::MemIo(
+                GuestPhysAddr::new(0xfec00000)..=GuestPhysAddr::new(0xfec010f0),
+            ),
+            //FIXME: this is actually the 1st HPET
+            DeviceRegion::MemIo(
+                GuestPhysAddr::new(0xfed00000)..=GuestPhysAddr::new(0xfed010f0),
+            ),
+        ]
+    }
+
+    fn on_event(&mut self, _event: Event) -> Result<()> {
+        Ok(())
+    }
+}

mythril/src/virtdev/lapic.rs

@@ -1,37 +1,8 @@
-use crate::error::Result;
-use crate::memory::GuestPhysAddr;
-use crate::virtdev::{DeviceRegion, EmulatedDevice, Event};
-use alloc::sync::Arc;
-use alloc::vec::Vec;
-use spin::RwLock;
-
 #[derive(Default)]
 pub struct LocalApic;
 
 impl LocalApic {
-    pub fn new() -> Arc<RwLock<Self>> {
-        Arc::new(RwLock::new(LocalApic::default()))
-    }
-}
-
-impl EmulatedDevice for LocalApic {
-    fn services(&self) -> Vec<DeviceRegion> {
-        vec![
-            DeviceRegion::MemIo(
-                GuestPhysAddr::new(0xfee00000)..=GuestPhysAddr::new(0xfee010f0),
-            ),
-            //FIXME: this is actually the 1st HPET
-            DeviceRegion::MemIo(
-                GuestPhysAddr::new(0xfed00000)..=GuestPhysAddr::new(0xfed010f0),
-            ),
-            //FIXME: this is actually the io apic
-            DeviceRegion::MemIo(
-                GuestPhysAddr::new(0xfec00000)..=GuestPhysAddr::new(0xfec010f0),
-            ),
-        ]
-    }
-
-    fn on_event(&mut self, _event: Event) -> Result<()> {
-        Ok(())
+    pub fn new() -> Self {
+        LocalApic {}
     }
 }

mythril/src/virtdev/mod.rs

@@ -16,6 +16,7 @@ pub mod com;
 pub mod debug;
 pub mod dma;
 pub mod ignore;
+pub mod ioapic;
 pub mod keyboard;
 pub mod lapic;
 pub mod pci;

mythril/src/vm.rs

@@ -24,6 +24,10 @@ use spin::RwLock;
 
 static BIOS_BLOB: &'static [u8] = include_bytes!("blob/bios.bin");
 
+//TODO(alschwalm): this should always be reported by the relevant MSR
+/// The location of the local apic in the guest address space
+pub const GUEST_LOCAL_APIC_ADDR: GuestPhysAddr = GuestPhysAddr::new(0xfee00000);
+
 static VIRTUAL_MACHINES: RoAfterInit<VirtualMachines> =
     RoAfterInit::uninitialized();
 
@@ -295,6 +299,11 @@ pub struct VirtualMachine {
     ///
     /// This will be shared by all `VCpu`s associated with this VM.
     pub guest_space: GuestAddressSpace,
+
+    /// The APIC access page
+    ///
+    /// See section 29.4 of the Intel software developer's manual
+    pub apic_access_page: Raw4kPage,
 }
 
 impl VirtualMachine {
@@ -309,11 +318,27 @@ impl VirtualMachine {
     ) -> Result<Arc<RwLock<Self>>> {
         let guest_space = Self::setup_ept(&config, info)?;
 
-        Ok(Arc::new(RwLock::new(Self {
+        let vm = Arc::new(RwLock::new(Self {
             id: id,
             config: config,
             guest_space: guest_space,
-        })))
+            apic_access_page: Raw4kPage([0u8; 4096]),
+        }));
+
+        // Map the guest local apic addr to the access page. This will be set in each
+        // core's vmcs
+        let apic_frame = memory::HostPhysFrame::from_start_address(
+            memory::HostPhysAddr::new(
+                vm.read().apic_access_page.as_ptr() as u64
+            ),
+        )?;
+        vm.write().guest_space.map_frame(
+            GUEST_LOCAL_APIC_ADDR,
+            apic_frame,
+            false,
+        )?;
+
+        Ok(vm)
     }
 
     pub fn dispatch_event(

mythril/src/vmexit.rs

@@ -116,7 +116,7 @@ pub enum ExitInformation {
     Pause,
     VmEntryMachineCheck,
     TprBelowThreshold,
-    ApicAccess,
+    ApicAccess(ApicAccessInformation),
     VirtualEio,
     AccessGdtridtr,
     AccessLdtrTr,
@@ -196,7 +196,9 @@ impl ExitReason {
             40 => ExitInformation::Pause,
             41 => ExitInformation::VmEntryMachineCheck,
             43 => ExitInformation::TprBelowThreshold,
-            44 => ExitInformation::ApicAccess,
+            44 => ExitInformation::ApicAccess(
+                ApicAccessInformation::from_active_vmcs(vmcs)?,
+            ),
             45 => ExitInformation::VirtualEio,
             46 => ExitInformation::AccessGdtridtr,
             47 => ExitInformation::AccessLdtrTr,
@@ -233,6 +235,49 @@ impl ExitReason {
     }
 }
 
+#[derive(Clone, Debug, TryFromPrimitive, PartialEq)]
+#[repr(u8)]
+pub enum ApicAccessKind {
+    LinearRead = 0,
+    LinearWrite = 1,
+    LinearFetch = 2,
+    LinearEventDelivery = 3,
+    PhysicalAccessDuringEvent = 10,
+    PhysicalAccessDuringFetch = 15,
+}
+
+#[derive(Clone, Debug)]
+pub struct ApicAccessInformation {
+    pub offset: Option<u16>,
+    pub kind: ApicAccessKind,
+    pub async_instr_exec: bool,
+}
+
+impl ExtendedExitInformation for ApicAccessInformation {
+    fn from_active_vmcs(vmcs: &vmcs::ActiveVmcs) -> Result<Self> {
+        let qualifier = vmcs.read_field(vmcs::VmcsField::ExitQualification)?;
+
+        let kind = ((qualifier >> 12) & 0b1111) as u8;
+        let kind = ApicAccessKind::try_from(kind)?;
+
+        let offset = if kind == ApicAccessKind::LinearRead
+            || kind == ApicAccessKind::LinearWrite
+            || kind == ApicAccessKind::LinearFetch
+            || kind == ApicAccessKind::LinearEventDelivery
+        {
+            Some((qualifier & 0xfff) as u16)
+        } else {
+            None
+        };
+
+        Ok(ApicAccessInformation {
+            kind: kind,
+            async_instr_exec: qualifier & (1 << 16) != 0,
+            offset: offset,
+        })
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct IoInstructionInformation {
     pub size: u8,

scripts/mythril.cfg

@@ -6,14 +6,14 @@
 	    "cpus": [0],
 	    "kernel": "kernel",
 	    "initramfs": "initramfs",
-            "cmdline": "disableapic earlyprintk=serial,0x3f8,115200 console=ttyS0 debug nokaslr noapic root=/dev/ram0"
+            "cmdline": "earlyprintk=serial,0x3f8,115200 console=ttyS0 debug nokaslr root=/dev/ram0"
         },
         {
 	    "memory": 256,
 	    "cpus": [1],
 	    "kernel": "kernel",
 	    "initramfs": "initramfs",
-            "cmdline": "disableapic earlyprintk=serial,0x3f8,115200 console=ttyS0 debug nokaslr noapic root=/dev/ram0"
+            "cmdline": "earlyprintk=serial,0x3f8,115200 console=ttyS0 debug nokaslr root=/dev/ram0"
         }
     ]
 }