Add inter-vm communications

This also adds a basic 'transfer console' implementation

Author: ALSchwalm

mythril/Cargo.lock

@@ -11,6 +11,7 @@ description = "A intel-focused hypervisor using VT-x/EPT"
 test = []
 
 [dependencies]
+arraydeque = { version = "0.4.5", default-features = false }
 arrayvec = { version = "0.5.1", default-features = false }
 bitflags = "1.2.0"
 byteorder = { version = "1", default-features = false }

mythril/Cargo.toml

@@ -188,8 +188,8 @@ impl LocalApic {
     }
 
     /// The APIC ID
-    pub fn id(&self) -> usize {
-        unsafe { msr::rdmsr(msr::IA32_X2APIC_APICID) as usize }
+    pub fn id(&self) -> u32 {
+        unsafe { msr::rdmsr(msr::IA32_X2APIC_APICID) as u32 }
     }
 
     /// The Logical APIC ID

mythril/src/apic.rs

@@ -168,13 +168,13 @@ _start:
     out 0x21, al
     out 0xa1, al
 
-    ; Set IRQ masks
-    mov al, 0xED                ; Unmask IRQ1 (ps2) and IRQ4 (COM1)
+    ;; Mask everything so the I/O APIC will work
+    mov al, 0xff
     out 0x21, al
-    mov al, 0xFF
+    mov al, 0xff
     out 0xA1, al
 
-    ; ACK any pending interrupts
+    ;; ACK any pending interrupts
     mov al, 0x20
     out 0x20, al
     out 0xa0, al

mythril/src/boot.S

@@ -1,5 +1,9 @@
 pub mod idt;
 
+pub const UART_VECTOR: u8 = 36;
+pub const TIMER_VECTOR: u8 = 48;
+pub const IPC_VECTOR: u8 = 49;
+
 pub unsafe fn enable_interrupts() {
     llvm_asm!("sti" :::: "volatile");
 }

mythril/src/interrupt/mod.rs

@@ -3,6 +3,7 @@ use crate::ap;
 use crate::apic;
 use crate::boot_info::BootInfo;
 use crate::interrupt;
+use crate::ioapic;
 use crate::linux;
 use crate::logger;
 use crate::memory;
@@ -14,7 +15,6 @@ use crate::vcpu;
 use crate::virtdev;
 use crate::vm;
 
-use alloc::collections::BTreeMap;
 use alloc::sync::Arc;
 use alloc::vec::Vec;
 use log::{debug, info};
@@ -29,7 +29,7 @@ extern "C" {
 
 // Temporary helper function to create a vm for a single core
 fn default_vm(
-    core: usize,
+    core: u32,
     mem: u64,
     info: &BootInfo,
     add_uart: bool,
@@ -47,7 +47,7 @@ fn default_vm(
     };
 
     let mut config =
-        vm::VirtualMachineConfig::new(vec![core as u8], mem, physical_config);
+        vm::VirtualMachineConfig::new(vec![core], mem, physical_config);
 
     // FIXME: When `map_bios` may return an error, log the error.
     config.map_bios("seabios.bin".into()).unwrap_or(());
@@ -132,7 +132,7 @@ fn default_vm(
     .unwrap();
     device_map.register_device(fw_cfg_builder.build()).unwrap();
 
-    vm::VirtualMachine::new(config, info).expect("Failed to create vm")
+    vm::VirtualMachine::new(core, config, info).expect("Failed to create vm")
 }
 
 #[no_mangle]
@@ -207,18 +207,27 @@ unsafe fn kmain(mut boot_info: BootInfo) -> ! {
         })
         .collect::<Vec<_>>();
 
+    ioapic::init_ioapics(&madt).expect("Failed to initialize IOAPICs");
+    ioapic::map_gsi_vector(4, interrupt::UART_VECTOR, 0)
+        .expect("Failed to map com0 gsi");
+
     percore::init_sections(apic_ids.len())
         .expect("Failed to initialize per-core sections");
 
-    let mut map = BTreeMap::new();
+    let mut builder = vm::VirtualMachineBuilder::new();
+
     for apic_id in apic_ids.iter() {
-        map.insert(
-            *apic_id as usize,
-            default_vm(*apic_id as usize, 256, &boot_info, *apic_id == 0),
-        );
+        builder
+            .insert_machine(default_vm(
+                *apic_id,
+                256,
+                &boot_info,
+                *apic_id == 0,
+            ))
+            .expect("Failed to insert new vm");
     }
 
-    vm::VM_MAP = Some(map);
+    vm::init_virtual_machines(builder.finalize());
 
     debug!("AP_STARTUP address: 0x{:x}", AP_STARTUP_ADDR);
 

mythril/src/kmain.rs

@@ -8,6 +8,7 @@
 
 use crate::apic;
 use crate::error::Result;
+use crate::interrupt;
 use crate::tsc;
 use crate::vcpu;
 use crate::{declare_per_core, get_per_core, get_per_core_mut};
@@ -16,8 +17,6 @@ use alloc::{collections::BTreeMap, vec};
 use core::ops::{Add, AddAssign, Sub, SubAssign};
 use core::time::Duration;
 
-const TIMER_VECTOR: u8 = 48;
-
 //TODO: should this just be stored as a VCPU member?
 declare_per_core! {
     static mut TIMER_WHEEL: Option<TimerWheel> = None;
@@ -331,7 +330,7 @@ impl TimerWheel {
         if let Some((when, _)) = soonest {
             unsafe {
                 apic::get_local_apic_mut()
-                    .schedule_interrupt(when, TIMER_VECTOR);
+                    .schedule_interrupt(when, interrupt::TIMER_VECTOR);
             }
         }
     }

mythril/src/time.rs

@@ -1,6 +1,8 @@
 use crate::apic;
 use crate::emulate;
 use crate::error::{self, Error, Result};
+use crate::interrupt;
+use crate::ioapic;
 use crate::memory::Raw4kPage;
 use crate::percore;
 use crate::registers::{GdtrBase, IdtrBase};
@@ -33,14 +35,11 @@ pub fn mp_entry_point() -> ! {
     }
 
     let vm = unsafe {
-        vm::VM_MAP
-            .as_ref()
-            .unwrap()
-            .get(&apic::get_local_apic().id())
-            .expect("Failed to find VM for core")
-            .clone()
+        let id = apic::get_local_apic().id();
+        vm::get_vm_for_apic_id(id)
+            .expect(&format!("Failed to find VM associated with 0x{:x}", id))
     };
-    let vcpu = VCpu::new(vm.clone()).expect("Failed to create vcpu");
+    let vcpu = VCpu::new(vm).expect("Failed to create vcpu");
     vcpu.launch().expect("Failed to launch vm")
 }
 
@@ -507,17 +506,26 @@ impl VCpu {
             vmexit::ExitInformation::InterruptWindow => {}
             vmexit::ExitInformation::ExternalInterrupt(info) => unsafe {
                 match info.vector {
-                    0x24 => self.handle_uart_keypress(&mut responses)?,
+                    interrupt::UART_VECTOR => {
+                        self.handle_uart_keypress(&mut responses)?
+                    }
+                    interrupt::IPC_VECTOR => {
+                        let msg =
+                            vm::recv_vm_msg().ok_or_else(|| Error::NotFound)?;
+                        match msg {
+                            vm::VirtualMachineMsg::GrantConsole(serial) => {
+                                let mut vm = self.vm.write();
+                                vm.config.physical_devices_mut().serial =
+                                    Some(serial);
+                            }
+                        }
+                    }
                     _ => (),
                 }
 
-                // Ack the interrupt via PIC or APIC based on the vector
-                if info.vector < 48 {
-                    //TODO: this should be a call to the physical PIC
-                    x86::io::outb(0x20, 0x20);
-                } else {
-                    apic::get_local_apic_mut().eoi();
-                }
+                // We don't use the PIC, so any interrupt must be ACKed through
+                // the local apic
+                apic::get_local_apic_mut().eoi();
             },
             _ => {
                 info!("{}", self.vmcs);
@@ -530,6 +538,38 @@ impl VCpu {
                 virtdev::DeviceEventResponse::Interrupt((vector, kind)) => {
                     self.inject_interrupt(vector, kind);
                 }
+                virtdev::DeviceEventResponse::NextConsole => {
+                    info!("Received Ctrl-a three times. Switching console to next VM");
+
+                    let mut vm = self.vm.write();
+                    let serial = vm
+                        .config
+                        .physical_devices_mut()
+                        .serial
+                        .take()
+                        .ok_or_else(|| Error::NotFound)?;
+                    let vmid = vm.id;
+                    drop(vm);
+
+                    let next_vmid = (vmid + 1) % vm::max_vm_id();
+
+                    vm::send_vm_msg(
+                        vm::VirtualMachineMsg::GrantConsole(serial),
+                        next_vmid,
+                    )?;
+
+                    //FIXME(alschwalm): this should use the vm's bsp apicid
+                    ioapic::map_gsi_vector(
+                        4,
+                        interrupt::UART_VECTOR,
+                        next_vmid as u8,
+                    )
+                    .map_err(|_| {
+                        Error::DeviceError(
+                            "Failed to update console GSI mapping".into(),
+                        )
+                    })?;
+                }
                 virtdev::DeviceEventResponse::GuestUartTransmitted(val) => {
                     let vm = self.vm.read();
                     if vm.config.physical_devices().serial.is_some() {

mythril/src/vcpu.rs

@@ -20,6 +20,8 @@ pub struct Uart8250 {
     _line_status_register: LsrFlags,
     _modem_status_register: u8,
     _scratch_register: u8,
+
+    ctrl_a_count: u8,
 }
 
 impl Uart8250 {
@@ -35,6 +37,7 @@ impl Uart8250 {
             _line_status_register: LsrFlags::empty(),
             _modem_status_register: 0,
             _scratch_register: 0,
+            ctrl_a_count: 0,
         }))
     }
 
@@ -65,6 +68,14 @@ impl EmulatedDevice for Uart8250 {
                         vcpu::InjectedInterruptType::ExternalInterrupt,
                     )),
                 );
+                if key == 0x01 {
+                    // ctrl+a
+                    self.ctrl_a_count += 1;
+                }
+                if self.ctrl_a_count == 3 {
+                    event.responses.push(DeviceEventResponse::NextConsole);
+                    self.ctrl_a_count = 0;
+                }
                 self.write(key)
             }
             DeviceEvent::PortRead(port, mut val) => {

mythril/src/virtdev/com.rs

@@ -43,6 +43,7 @@ pub enum DeviceEvent<'a> {
 #[derive(Debug)]
 pub enum DeviceEventResponse {
     GuestUartTransmitted(u8),
+    NextConsole,
     Interrupt((u8, vcpu::InjectedInterruptType)),
 }