From eeeb45bbb9150701c99c9f955461b154fd45fc49 Mon Sep 17 00:00:00 2001 From: Michael Zhao Date: Tue, 9 Jun 2020 14:17:42 +0800 Subject: [PATCH] vmm: Enable device manager for AArch64 Screened IO bus because it is not for AArch64. Enabled Serial, RTC and Virtio devices with MMIO transport option. Signed-off-by: Michael Zhao --- vmm/src/cpu.rs | 9 +- vmm/src/device_manager.rs | 289 ++++++++++++++++++++++++++++++-------- 2 files changed, 241 insertions(+), 57 deletions(-) diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs index 05f6b83c5..46af3d54e 100644 --- a/vmm/src/cpu.rs +++ b/vmm/src/cpu.rs @@ -230,7 +230,7 @@ struct InterruptSourceOverride { pub struct Vcpu { fd: VcpuFd, id: u8, - #[cfg_attr(target_arch = "aarch64", allow(dead_code))] + #[cfg(target_arch = "x86_64")] io_bus: Arc, mmio_bus: Arc, #[cfg_attr(target_arch = "aarch64", allow(dead_code))] @@ -267,7 +267,7 @@ impl Vcpu { pub fn new( id: u8, fd: &Arc, - io_bus: Arc, + #[cfg(target_arch = "x86_64")] io_bus: Arc, mmio_bus: Arc, interrupt_controller: Option>>, creation_ts: std::time::Instant, @@ -277,6 +277,7 @@ impl Vcpu { Ok(Arc::new(Mutex::new(Vcpu { fd: kvm_vcpu, id, + #[cfg(target_arch = "x86_64")] io_bus, mmio_bus, interrupt_controller, @@ -504,6 +505,7 @@ impl Snapshottable for Vcpu { pub struct CpuManager { boot_vcpus: u8, max_vcpus: u8, + #[cfg(target_arch = "x86_64")] io_bus: Arc, #[cfg_attr(target_arch = "aarch64", allow(dead_code))] mmio_bus: Arc, @@ -657,6 +659,7 @@ impl CpuManager { let cpu_manager = Arc::new(Mutex::new(CpuManager { boot_vcpus: config.boot_vcpus, max_vcpus: config.max_vcpus, + #[cfg(target_arch = "x86_64")] io_bus: device_manager.io_bus().clone(), mmio_bus: device_manager.mmio_bus().clone(), interrupt_controller: device_manager.interrupt_controller().clone(), @@ -679,6 +682,7 @@ impl CpuManager { .allocate_io_addresses(Some(GuestAddress(0x0cd8)), 0x8, None) .ok_or(Error::AllocateIOPort)?; + #[cfg(target_arch = "x86_64")] cpu_manager .lock() .unwrap() @@ -742,6 +746,7 @@ impl CpuManager { let vcpu = Vcpu::new( cpu_id, &self.fd, + #[cfg(target_arch = "x86_64")] self.io_bus.clone(), self.mmio_bus.clone(), interrupt_controller, diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index da232191b..6f551881b 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -22,16 +22,20 @@ use crate::{device_node, DEVICE_MANAGER_SNAPSHOT_ID}; #[cfg(feature = "acpi")] use acpi_tables::{aml, aml::Aml}; use anyhow::anyhow; +#[cfg(target_arch = "aarch64")] +use arch::aarch64::DeviceInfoForFDT; #[cfg(feature = "acpi")] use arch::layout; #[cfg(target_arch = "x86_64")] use arch::layout::{APIC_START, IOAPIC_SIZE, IOAPIC_START}; #[cfg(target_arch = "aarch64")] +use arch::DeviceType; +#[cfg(target_arch = "aarch64")] use devices::gic; #[cfg(target_arch = "x86_64")] use devices::ioapic; use devices::{ - interrupt_controller, interrupt_controller::InterruptController, BusDevice, + interrupt_controller, interrupt_controller::InterruptController, legacy::Serial, BusDevice, HotPlugNotificationFlags, }; use kvm_ioctls::*; @@ -382,7 +386,7 @@ pub fn get_win_size() -> (u16, u16) { #[derive(Default)] pub struct Console { // Serial port on 0x3f8 - serial: Option>>, + serial: Option>>, console_input: Option>, input_enabled: bool, } @@ -421,6 +425,7 @@ impl Console { struct AddressManager { allocator: Arc>, + #[cfg(target_arch = "x86_64")] io_bus: Arc, mmio_bus: Arc, vm_fd: Arc, @@ -630,6 +635,28 @@ struct DeviceManagerState { device_id_cnt: Wrapping, } +/// Private structure for storing information about the MMIO device registered at some address on the bus. +#[derive(Clone, Debug)] +#[cfg(target_arch = "aarch64")] +pub struct MMIODeviceInfo { + addr: u64, + irq: u32, + len: u64, +} + +#[cfg(target_arch = "aarch64")] +impl DeviceInfoForFDT for MMIODeviceInfo { + fn addr(&self) -> u64 { + self.addr + } + fn irq(&self) -> u32 { + self.irq + } + fn length(&self) -> u64 { + self.len + } +} + pub struct DeviceManager { // Manage address space related to devices address_manager: Arc, @@ -713,8 +740,13 @@ pub struct DeviceManager { // Exit event #[cfg(feature = "acpi")] exit_evt: EventFd, + // Reset event + #[cfg(target_arch = "x86_64")] reset_evt: EventFd, + + #[cfg(target_arch = "aarch64")] + id_to_dev_info: HashMap<(DeviceType, String), MMIODeviceInfo>, } impl DeviceManager { @@ -723,13 +755,14 @@ impl DeviceManager { config: Arc>, memory_manager: Arc>, _exit_evt: &EventFd, - reset_evt: &EventFd, + #[cfg_attr(target_arch = "aarch64", allow(unused_variables))] reset_evt: &EventFd, vmm_path: PathBuf, ) -> DeviceManagerResult>> { let device_tree = Arc::new(Mutex::new(DeviceTree::new())); let address_manager = Arc::new(AddressManager { allocator: memory_manager.lock().unwrap().allocator(), + #[cfg(target_arch = "x86_64")] io_bus: Arc::new(devices::Bus::new()), mmio_bus: Arc::new(devices::Bus::new()), vm_fd: vm_fd.clone(), @@ -789,7 +822,10 @@ impl DeviceManager { device_tree, #[cfg(feature = "acpi")] exit_evt: _exit_evt.try_clone().map_err(DeviceManagerError::EventFd)?, + #[cfg(target_arch = "x86_64")] reset_evt: reset_evt.try_clone().map_err(DeviceManagerError::EventFd)?, + #[cfg(target_arch = "aarch64")] + id_to_dev_info: HashMap::new(), }; #[cfg(feature = "acpi")] @@ -846,12 +882,16 @@ impl DeviceManager { ) .map_err(DeviceManagerError::BusError)?; + #[cfg(target_arch = "x86_64")] self.add_legacy_devices( self.reset_evt .try_clone() .map_err(DeviceManagerError::EventFd)?, )?; + #[cfg(target_arch = "aarch64")] + self.add_legacy_devices(&legacy_interrupt_manager)?; + #[cfg(feature = "acpi")] { self.ged_notification_device = self.add_acpi_devices( @@ -895,6 +935,12 @@ impl DeviceManager { Ok(()) } + #[cfg(target_arch = "aarch64")] + /// Gets the information of the devices registered up to some point in time. + pub fn get_device_info(&self) -> &HashMap<(DeviceType, String), MMIODeviceInfo> { + &self.id_to_dev_info + } + #[allow(unused_variables)] fn add_pci_devices( &mut self, @@ -980,6 +1026,7 @@ impl DeviceManager { let pci_config_io = Arc::new(Mutex::new(PciConfigIo::new(Arc::clone(&pci_bus)))); self.bus_devices .push(Arc::clone(&pci_config_io) as Arc>); + #[cfg(target_arch = "x86_64")] self.address_manager .io_bus .insert(pci_config_io, 0xcf8, 0x8) @@ -1144,6 +1191,7 @@ impl DeviceManager { Ok(Some(ged_device)) } + #[cfg(target_arch = "x86_64")] fn add_legacy_devices(&mut self, reset_evt: EventFd) -> DeviceManagerResult<()> { // Add a shutdown device (i8042) let i8042 = Arc::new(Mutex::new(devices::legacy::I8042Device::new(reset_evt))); @@ -1200,6 +1248,160 @@ impl DeviceManager { Ok(()) } + #[cfg(target_arch = "aarch64")] + fn add_legacy_devices( + &mut self, + interrupt_manager: &Arc>, + ) -> DeviceManagerResult<()> { + // Add a RTC device + let rtc_irq = self + .address_manager + .allocator + .lock() + .unwrap() + .allocate_irq() + .unwrap(); + + let interrupt_group = interrupt_manager + .create_group(LegacyIrqGroupConfig { + irq: rtc_irq as InterruptIndex, + }) + .map_err(DeviceManagerError::CreateInterruptGroup)?; + + let rtc_device = Arc::new(Mutex::new(devices::legacy::RTC::new(interrupt_group))); + + self.bus_devices + .push(Arc::clone(&rtc_device) as Arc>); + + let addr = GuestAddress(arch::layout::LEGACY_RTC_MAPPED_IO_START); + + self.address_manager + .mmio_bus + .insert(rtc_device.clone(), addr.0, MMIO_LEN) + .map_err(DeviceManagerError::BusError)?; + + self.id_to_dev_info.insert( + (DeviceType::RTC, "rtc".to_string()), + MMIODeviceInfo { + addr: addr.0, + len: MMIO_LEN, + irq: rtc_irq, + }, + ); + + Ok(()) + } + + #[cfg(target_arch = "x86_64")] + fn add_serial_device( + &mut self, + interrupt_manager: &Arc>, + serial_writer: Option>, + ) -> DeviceManagerResult>> { + // Serial is tied to IRQ #4 + let serial_irq = 4; + + let id = String::from(SERIAL_DEVICE_NAME_PREFIX); + + let interrupt_group = interrupt_manager + .create_group(LegacyIrqGroupConfig { + irq: serial_irq as InterruptIndex, + }) + .map_err(DeviceManagerError::CreateInterruptGroup)?; + + let serial = Arc::new(Mutex::new(Serial::new( + id.clone(), + interrupt_group, + serial_writer, + ))); + + self.bus_devices + .push(Arc::clone(&serial) as Arc>); + + self.address_manager + .allocator + .lock() + .unwrap() + .allocate_io_addresses(Some(GuestAddress(0x3f8)), 0x8, None) + .ok_or(DeviceManagerError::AllocateIOPort)?; + + self.address_manager + .io_bus + .insert(serial.clone(), 0x3f8, 0x8) + .map_err(DeviceManagerError::BusError)?; + + // Fill the device tree with a new node. In case of restore, we + // know there is nothing to do, so we can simply override the + // existing entry. + self.device_tree + .lock() + .unwrap() + .insert(id.clone(), device_node!(id, serial)); + + Ok(serial) + } + + #[cfg(target_arch = "aarch64")] + fn add_serial_device( + &mut self, + interrupt_manager: &Arc>, + serial_writer: Option>, + ) -> DeviceManagerResult>> { + let id = String::from(SERIAL_DEVICE_NAME_PREFIX); + + let serial_irq = self + .address_manager + .allocator + .lock() + .unwrap() + .allocate_irq() + .unwrap(); + + let interrupt_group = interrupt_manager + .create_group(LegacyIrqGroupConfig { + irq: serial_irq as InterruptIndex, + }) + .map_err(DeviceManagerError::CreateInterruptGroup)?; + + let serial = Arc::new(Mutex::new(Serial::new( + id.clone(), + interrupt_group, + serial_writer, + ))); + + self.bus_devices + .push(Arc::clone(&serial) as Arc>); + + let addr = GuestAddress(arch::layout::LEGACY_SERIAL_MAPPED_IO_START); + + self.address_manager + .mmio_bus + .insert(serial.clone(), addr.0, MMIO_LEN) + .map_err(DeviceManagerError::BusError)?; + + self.id_to_dev_info.insert( + (DeviceType::Serial, DeviceType::Serial.to_string()), + MMIODeviceInfo { + addr: addr.0, + len: MMIO_LEN, + irq: serial_irq, + }, + ); + + self.cmdline_additions + .push(format!("earlycon=uart,mmio,0x{:08x}", addr.0)); + + // Fill the device tree with a new node. In case of restore, we + // know there is nothing to do, so we can simply override the + // existing entry. + self.device_tree + .lock() + .unwrap() + .insert(id.clone(), device_node!(id, serial)); + + Ok(serial) + } + fn add_console_device( &mut self, interrupt_manager: &Arc>, @@ -1215,47 +1417,7 @@ impl DeviceManager { ConsoleOutputMode::Off | ConsoleOutputMode::Null => None, }; let serial = if serial_config.mode != ConsoleOutputMode::Off { - // Serial is tied to IRQ #4 - let serial_irq = 4; - - let id = String::from(SERIAL_DEVICE_NAME_PREFIX); - - let interrupt_group = interrupt_manager - .create_group(LegacyIrqGroupConfig { - irq: serial_irq as InterruptIndex, - }) - .map_err(DeviceManagerError::CreateInterruptGroup)?; - - let serial = Arc::new(Mutex::new(devices::legacy::Serial::new( - id.clone(), - interrupt_group, - serial_writer, - ))); - - self.bus_devices - .push(Arc::clone(&serial) as Arc>); - - self.address_manager - .allocator - .lock() - .unwrap() - .allocate_io_addresses(Some(GuestAddress(0x3f8)), 0x8, None) - .ok_or(DeviceManagerError::AllocateIOPort)?; - - self.address_manager - .io_bus - .insert(serial.clone(), 0x3f8, 0x8) - .map_err(DeviceManagerError::BusError)?; - - // Fill the device tree with a new node. In case of restore, we - // know there is nothing to do, so we can simply override the - // existing entry. - self.device_tree - .lock() - .unwrap() - .insert(id.clone(), device_node!(id, serial)); - - Some(serial) + Some(self.add_serial_device(interrupt_manager, serial_writer)?) } else { None }; @@ -2432,6 +2594,7 @@ impl DeviceManager { pci.register_mapping( virtio_pci_device.clone(), + #[cfg(target_arch = "x86_64")] self.address_manager.io_bus.as_ref(), self.address_manager.mmio_bus.as_ref(), bars.clone(), @@ -2533,6 +2696,30 @@ impl DeviceManager { (base.raw_value(), size) }; + let irq_num = if let Some(irq) = mmio_irq { + irq + } else { + self.address_manager + .allocator + .lock() + .unwrap() + .allocate_irq() + .ok_or(DeviceManagerError::AllocateIrq)? + }; + + #[cfg(target_arch = "aarch64")] + { + let device_type = virtio_device.lock().unwrap().device_type(); + self.id_to_dev_info.insert( + (DeviceType::Virtio(device_type), virtio_device_id), + MMIODeviceInfo { + addr: mmio_base, + len: mmio_size, + irq: irq_num, + }, + ); + } + let memory = self.memory_manager.lock().unwrap().guest_memory(); let mut mmio_device = vm_virtio::transport::MmioDevice::new(id.clone(), memory, virtio_device) @@ -2546,17 +2733,6 @@ impl DeviceManager { .map_err(DeviceManagerError::RegisterIoevent)?; } - let irq_num = if let Some(irq) = mmio_irq { - irq - } else { - self.address_manager - .allocator - .lock() - .unwrap() - .allocate_irq() - .ok_or(DeviceManagerError::AllocateIrq)? - }; - let interrupt_group = interrupt_manager .create_group(LegacyIrqGroupConfig { irq: irq_num as InterruptIndex, @@ -2573,6 +2749,7 @@ impl DeviceManager { .insert(mmio_device_arc.clone(), mmio_base, MMIO_LEN) .map_err(DeviceManagerError::BusError)?; + #[cfg(target_arch = "x86_64")] self.cmdline_additions.push(format!( "virtio_mmio.device={}K@0x{:08x}:{}", mmio_size / 1024, @@ -2592,6 +2769,7 @@ impl DeviceManager { Ok(()) } + #[cfg(target_arch = "x86_64")] pub fn io_bus(&self) -> &Arc { &self.address_manager.io_bus } @@ -2810,6 +2988,7 @@ impl DeviceManager { .remove_by_device(&pci_device) .map_err(DeviceManagerError::RemoveDeviceFromPciBus)?; + #[cfg(target_arch = "x86_64")] // Remove the device from the IO bus self.io_bus() .remove_by_device(&bus_device)