From 987f82152e702b06e55f17997a71ab34eda37704 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 29 Apr 2020 15:18:04 +0200 Subject: [PATCH] vmm: Store and restore virtio-mmio resources Based on the device tree, retrieve the resources associated with a virtio-mmio device to restore it at the right location in guest address space. Also, the IRQ number is correctly restored. Signed-off-by: Sebastien Boeuf --- vm-device/src/lib.rs | 4 +- vmm/src/device_manager.rs | 123 ++++++++++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 27 deletions(-) diff --git a/vm-device/src/lib.rs b/vm-device/src/lib.rs index f7914f552..3287f47ba 100644 --- a/vm-device/src/lib.rs +++ b/vm-device/src/lib.rs @@ -10,7 +10,7 @@ use vm_memory::{ }; /// Type of Message Singaled Interrupt -#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum MsiIrqType { /// PCI MSI IRQ numbers. PciMsi, @@ -22,7 +22,7 @@ pub enum MsiIrqType { /// Enumeration for device resources. #[allow(missing_docs)] -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub enum Resource { /// IO Port address range. PioAddressRange { base: u16, size: u16 }, diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index 59fbd3346..e1877e80a 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -322,6 +322,12 @@ pub enum DeviceManagerError { /// Could not find a MMIO range. MmioRangeAllocation, + + /// Resource was already found. + ResourceAlreadyExists, + + /// Expected resources for virtio-mmio could not be found. + MissingVirtioMmioResources, } pub type DeviceManagerResult = result::Result; @@ -2129,12 +2135,50 @@ impl DeviceManager { ) -> DeviceManagerResult<()> { let id = format!("{}-{}", VIRTIO_MMIO_DEVICE_NAME_PREFIX, virtio_device_id); - // Add the new virtio-mmio node to the device tree. - let node = Node { - child: Some(virtio_device_id.clone()), - ..Default::default() + // Look for the id in the device tree. If it can be found, that means + // the device is being restored, otherwise it's created from scratch. + let (mmio_range, mmio_irq) = if let Some(node) = self.device_tree.get(&id) { + debug!("Restoring virtio-mmio {} resources", id); + + let mut mmio_range: Option<(u64, u64)> = None; + let mut mmio_irq: Option = None; + for resource in node.resources.iter() { + match resource { + Resource::MmioAddressRange { base, size } => { + if mmio_range.is_some() { + return Err(DeviceManagerError::ResourceAlreadyExists); + } + + mmio_range = Some((*base, *size)); + } + Resource::LegacyIrq(irq) => { + if mmio_irq.is_some() { + return Err(DeviceManagerError::ResourceAlreadyExists); + } + + mmio_irq = Some(*irq); + } + _ => { + error!("Unexpected resource {:?} for {}", resource, id); + } + } + } + + if mmio_range.is_none() || mmio_irq.is_none() { + return Err(DeviceManagerError::MissingVirtioMmioResources); + } + + (mmio_range, mmio_irq) + } else { + // Add the new virtio-mmio node to the device tree. + let node = Node { + child: Some(virtio_device_id.clone()), + ..Default::default() + }; + self.device_tree.insert(id.clone(), node); + + (None, None) }; - self.device_tree.insert(id.clone(), node); // Update the existing virtio node by setting the parent. if let Some(node) = self.device_tree.get_mut(&virtio_device_id) { @@ -2143,19 +2187,34 @@ impl DeviceManager { return Err(DeviceManagerError::MissingNode); } - let mmio_base = self - .address_manager - .allocator - .lock() - .unwrap() - .allocate_mmio_addresses(None, MMIO_LEN, Some(MMIO_LEN)) - .ok_or(DeviceManagerError::MmioRangeAllocation)?; + let (mmio_base, mmio_size) = if let Some((base, size)) = mmio_range { + self.address_manager + .allocator + .lock() + .unwrap() + .allocate_mmio_addresses(Some(GuestAddress(base)), size, Some(size)) + .ok_or(DeviceManagerError::MmioRangeAllocation)?; + + (base, size) + } else { + let size = MMIO_LEN; + let base = self + .address_manager + .allocator + .lock() + .unwrap() + .allocate_mmio_addresses(None, size, Some(size)) + .ok_or(DeviceManagerError::MmioRangeAllocation)?; + + (base.raw_value(), size) + }; let memory = self.memory_manager.lock().unwrap().guest_memory(); - let mut mmio_device = vm_virtio::transport::MmioDevice::new(id, memory, virtio_device) - .map_err(DeviceManagerError::VirtioDevice)?; + let mut mmio_device = + vm_virtio::transport::MmioDevice::new(id.clone(), memory, virtio_device) + .map_err(DeviceManagerError::VirtioDevice)?; - for (i, (event, addr)) in mmio_device.ioeventfds(mmio_base.0).iter().enumerate() { + for (i, (event, addr)) in mmio_device.ioeventfds(mmio_base).iter().enumerate() { let io_addr = IoEventAddress::Mmio(*addr); self.address_manager .vm_fd @@ -2163,13 +2222,16 @@ impl DeviceManager { .map_err(DeviceManagerError::RegisterIoevent)?; } - let irq_num = self - .address_manager - .allocator - .lock() - .unwrap() - .allocate_irq() - .ok_or(DeviceManagerError::AllocateIrq)?; + let irq_num = if let Some(irq) = mmio_irq { + irq + } else { + self.address_manager + .allocator + .lock() + .unwrap() + .allocate_irq() + .ok_or(DeviceManagerError::AllocateIrq)? + }; let interrupt_group = interrupt_manager .create_group(LegacyIrqGroupConfig { @@ -2179,18 +2241,29 @@ impl DeviceManager { mmio_device.assign_interrupt(interrupt_group); + // Update the device tree with correct resource information. + if let Some(node) = self.device_tree.get_mut(&id) { + node.resources.push(Resource::MmioAddressRange { + base: mmio_base, + size: mmio_size, + }); + node.resources.push(Resource::LegacyIrq(irq_num)); + } else { + return Err(DeviceManagerError::MissingNode); + } + let mmio_device_arc = Arc::new(Mutex::new(mmio_device)); self.bus_devices .push(Arc::clone(&mmio_device_arc) as Arc>); self.address_manager .mmio_bus - .insert(mmio_device_arc.clone(), mmio_base.0, MMIO_LEN) + .insert(mmio_device_arc.clone(), mmio_base, MMIO_LEN) .map_err(DeviceManagerError::BusError)?; self.cmdline_additions.push(format!( "virtio_mmio.device={}K@0x{:08x}:{}", - MMIO_LEN / 1024, - mmio_base.0, + mmio_size / 1024, + mmio_base, irq_num ));