diff --git a/Cargo.lock b/Cargo.lock index 999153d9c..aeff21b9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,6 +144,7 @@ dependencies = [ "versionize_derive", "vhdx", "virtio-bindings", + "virtio-queue", "vm-memory", "vm-virtio", "vmm-sys-util", @@ -607,6 +608,7 @@ dependencies = [ "versionize", "versionize_derive", "virtio-bindings", + "virtio-queue", "vm-memory", "vm-virtio", "vmm-sys-util", @@ -1212,6 +1214,7 @@ dependencies = [ "log", "vhost", "virtio-bindings", + "virtio-queue", "vm-memory", "vm-virtio", "vmm-sys-util", @@ -1285,6 +1288,7 @@ dependencies = [ "versionize_derive", "vhost", "virtio-bindings", + "virtio-queue", "vm-allocator", "vm-device", "vm-memory", @@ -1293,6 +1297,15 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "virtio-queue" +version = "0.1.0" +dependencies = [ + "log", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "vm-allocator" version = "0.1.0" @@ -1352,6 +1365,7 @@ version = "0.1.0" dependencies = [ "log", "virtio-bindings", + "virtio-queue", "vm-memory", ] @@ -1393,6 +1407,7 @@ dependencies = [ "vfio_user", "vhdx", "virtio-devices", + "virtio-queue", "vm-allocator", "vm-device", "vm-memory", diff --git a/Cargo.toml b/Cargo.toml index 71a76826a..53c2bc1d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ members = [ "vhost_user_block", "vhost_user_net", "virtio-devices", + "virtio-queue", "vmm", "vm-allocator", "vm-device", diff --git a/block_util/Cargo.toml b/block_util/Cargo.toml index 6c24cfac8..9dc4cca74 100644 --- a/block_util/Cargo.toml +++ b/block_util/Cargo.toml @@ -18,6 +18,7 @@ versionize = "0.1.6" versionize_derive = "0.1.4" vhdx = { path = "../vhdx" } virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] } +virtio-queue = { path = "../virtio-queue" } vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] } vm-virtio = { path = "../vm-virtio" } vmm-sys-util = "0.9.0" diff --git a/block_util/src/lib.rs b/block_util/src/lib.rs index 2da33de63..cc39ee586 100644 --- a/block_util/src/lib.rs +++ b/block_util/src/lib.rs @@ -36,11 +36,11 @@ use std::sync::{Arc, Mutex}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; use virtio_bindings::bindings::virtio_blk::*; +use virtio_queue::DescriptorChain; use vm_memory::{ bitmap::AtomicBitmap, bitmap::Bitmap, ByteValued, Bytes, GuestAddress, GuestMemory, - GuestMemoryError, + GuestMemoryAtomic, GuestMemoryError, }; -use vm_virtio::DescriptorChain; use vmm_sys_util::eventfd::EventFd; type GuestMemoryMmap = vm_memory::GuestMemoryMmap; @@ -179,25 +179,32 @@ pub struct Request { impl Request { pub fn parse( - avail_desc: &DescriptorChain, - mem: &GuestMemoryMmap, + desc_chain: &mut DescriptorChain>, ) -> result::Result { + let hdr_desc = desc_chain + .next() + .ok_or(Error::DescriptorChainTooShort) + .map_err(|e| { + error!("Missing head descriptor"); + e + })?; + // The head contains the request type which MUST be readable. - if avail_desc.is_write_only() { + if hdr_desc.is_write_only() { return Err(Error::UnexpectedWriteOnlyDescriptor); } let mut req = Request { - request_type: request_type(mem, avail_desc.addr)?, - sector: sector(mem, avail_desc.addr)?, + request_type: request_type(desc_chain.memory(), hdr_desc.addr())?, + sector: sector(desc_chain.memory(), hdr_desc.addr())?, data_descriptors: Vec::new(), status_addr: GuestAddress(0), writeback: true, }; let status_desc; - let mut desc = avail_desc - .next_descriptor() + let mut desc = desc_chain + .next() .ok_or(Error::DescriptorChainTooShort) .map_err(|e| { error!("Only head descriptor present: request = {:?}", req); @@ -222,9 +229,9 @@ impl Request { if !desc.is_write_only() && req.request_type == RequestType::GetDeviceId { return Err(Error::UnexpectedReadOnlyDescriptor); } - req.data_descriptors.push((desc.addr, desc.len)); - desc = desc - .next_descriptor() + req.data_descriptors.push((desc.addr(), desc.len())); + desc = desc_chain + .next() .ok_or(Error::DescriptorChainTooShort) .map_err(|e| { error!("DescriptorChain corrupted: request = {:?}", req); @@ -239,11 +246,11 @@ impl Request { return Err(Error::UnexpectedReadOnlyDescriptor); } - if status_desc.len < 1 { + if status_desc.len() < 1 { return Err(Error::DescriptorLengthTooSmall); } - req.status_addr = status_desc.addr; + req.status_addr = status_desc.addr(); Ok(req) } diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 164672e7a..1c13c8601 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -105,6 +105,7 @@ dependencies = [ "versionize_derive", "vhdx", "virtio-bindings", + "virtio-queue", "vm-memory", "vm-virtio", "vmm-sys-util", @@ -178,6 +179,7 @@ dependencies = [ "seccompiler", "vhdx", "virtio-devices", + "virtio-queue", "vm-memory", "vm-virtio", "vmm-sys-util", @@ -401,6 +403,7 @@ dependencies = [ "versionize", "versionize_derive", "virtio-bindings", + "virtio-queue", "vm-memory", "vm-virtio", "vmm-sys-util", @@ -786,6 +789,7 @@ dependencies = [ "versionize_derive", "vhost", "virtio-bindings", + "virtio-queue", "vm-allocator", "vm-device", "vm-memory", @@ -794,6 +798,16 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "virtio-queue" +version = "0.1.0" +source = "git+https://github.com/sboeuf/vm-virtio?branch=fork_vm_virtio#223246e132e49bba58adb3d4156e25ff977c0eb1" +dependencies = [ + "log", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "vm-allocator" version = "0.1.0" @@ -853,6 +867,7 @@ version = "0.1.0" dependencies = [ "log", "virtio-bindings", + "virtio-queue", "vm-memory", ] @@ -893,6 +908,7 @@ dependencies = [ "vfio_user", "vhdx", "virtio-devices", + "virtio-queue", "vm-allocator", "vm-device", "vm-memory", diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 154da014d..1b685ebea 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -16,6 +16,7 @@ qcow = { path = "../qcow" } seccompiler = "0.2.0" vhdx = { path = "../vhdx" } virtio-devices = { path = "../virtio-devices" } +virtio-queue = { path = "../virtio-queue" } vmm-sys-util = "0.9.0" vm-virtio = { path = "../vm-virtio" } vm-memory = "0.6.0" diff --git a/fuzz/fuzz_targets/block.rs b/fuzz/fuzz_targets/block.rs index 076ff9dc7..2c7d1c4b6 100644 --- a/fuzz/fuzz_targets/block.rs +++ b/fuzz/fuzz_targets/block.rs @@ -15,10 +15,12 @@ use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::path::PathBuf; use std::sync::Arc; use virtio_devices::{Block, VirtioDevice, VirtioInterrupt, VirtioInterruptType}; -use vm_memory::{Bytes, GuestAddress, GuestMemoryAtomic, GuestMemoryMmap}; -use vm_virtio::Queue; +use virtio_queue::{Queue, QueueState}; +use vm_memory::{bitmap::AtomicBitmap, Bytes, GuestAddress, GuestMemoryAtomic}; use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK}; +type GuestMemoryMmap = vm_memory::GuestMemoryMmap; + const MEM_SIZE: u64 = 256 * 1024 * 1024; const DESC_SIZE: u64 = 16; // Bytes in one virtio descriptor. const QUEUE_SIZE: u16 = 16; // Max entries in the queue. @@ -73,10 +75,14 @@ fuzz_target!(|bytes| { return; } - let mut q = Queue::new(QUEUE_SIZE); - q.ready = true; - q.size = QUEUE_SIZE / 2; - q.max_size = QUEUE_SIZE; + let guest_memory = GuestMemoryAtomic::new(mem); + + let mut q = Queue::< + GuestMemoryAtomic, + QueueState>, + >::new(guest_memory.clone(), QUEUE_SIZE); + q.state.ready = true; + q.state.size = QUEUE_SIZE / 2; let queue_evts: Vec = vec![EventFd::new(0).unwrap()]; let queue_fd = queue_evts[0].as_raw_fd(); @@ -102,7 +108,7 @@ fuzz_target!(|bytes| { block .activate( - GuestMemoryAtomic::new(mem), + guest_memory, Arc::new(NoopVirtioInterrupt {}), vec![q], queue_evts, @@ -134,7 +140,7 @@ impl VirtioInterrupt for NoopVirtioInterrupt { fn trigger( &self, _int_type: &VirtioInterruptType, - _queue: Option<&Queue>, + _queue: Option<&Queue>>, ) -> std::result::Result<(), std::io::Error> { Ok(()) } diff --git a/net_util/Cargo.toml b/net_util/Cargo.toml index a5e7b3ab2..93caa135d 100644 --- a/net_util/Cargo.toml +++ b/net_util/Cargo.toml @@ -15,6 +15,7 @@ serde = "1.0.130" versionize = "0.1.6" versionize_derive = "0.1.4" virtio-bindings = "0.1.0" +virtio-queue = { path = "../virtio-queue" } vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] } vm-virtio = { path = "../vm-virtio" } vmm-sys-util = "0.9.0" diff --git a/net_util/src/ctrl_queue.rs b/net_util/src/ctrl_queue.rs index b820afdbb..ec87480d0 100644 --- a/net_util/src/ctrl_queue.rs +++ b/net_util/src/ctrl_queue.rs @@ -12,17 +12,25 @@ use virtio_bindings::bindings::virtio_net::{ VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, VIRTIO_NET_F_GUEST_UFO, VIRTIO_NET_OK, }; -use vm_memory::{ByteValued, Bytes, GuestMemoryError}; -use vm_virtio::Queue; +use virtio_queue::Queue; +use vm_memory::{ByteValued, Bytes, GuestMemoryAtomic, GuestMemoryError}; #[derive(Debug)] pub enum Error { /// Read queue failed. GuestMemory(GuestMemoryError), + /// No control header descriptor + NoControlHeaderDescriptor, /// No queue pairs number. NoQueuePairsDescriptor, /// No status descriptor NoStatusDescriptor, + /// Failed adding used index + QueueAddUsed(virtio_queue::Error), + /// Failed creating an iterator over the queue + QueueIterator(virtio_queue::Error), + /// Failed enabling notification for the queue + QueueEnableNotification(virtio_queue::Error), } type Result = std::result::Result; @@ -45,22 +53,26 @@ impl CtrlQueue { CtrlQueue { taps } } - pub fn process(&mut self, mem: &GuestMemoryMmap, queue: &mut Queue) -> Result { + pub fn process( + &mut self, + queue: &mut Queue>, + ) -> Result { let mut used_desc_heads = Vec::new(); - for avail_desc in queue.iter(mem) { - let ctrl_hdr: ControlHeader = - mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?; - let data_desc = avail_desc - .next_descriptor() - .ok_or(Error::NoQueuePairsDescriptor)?; - let status_desc = data_desc - .next_descriptor() - .ok_or(Error::NoStatusDescriptor)?; + for mut desc_chain in queue.iter().map_err(Error::QueueIterator)? { + let ctrl_desc = desc_chain.next().ok_or(Error::NoControlHeaderDescriptor)?; + + let ctrl_hdr: ControlHeader = desc_chain + .memory() + .read_obj(ctrl_desc.addr()) + .map_err(Error::GuestMemory)?; + let data_desc = desc_chain.next().ok_or(Error::NoQueuePairsDescriptor)?; + let status_desc = desc_chain.next().ok_or(Error::NoStatusDescriptor)?; let ok = match u32::from(ctrl_hdr.class) { VIRTIO_NET_CTRL_MQ => { - let queue_pairs = mem - .read_obj::(data_desc.addr) + let queue_pairs = desc_chain + .memory() + .read_obj::(data_desc.addr()) .map_err(Error::GuestMemory)?; if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET { warn!("Unsupported command: {}", ctrl_hdr.cmd); @@ -76,8 +88,9 @@ impl CtrlQueue { } } VIRTIO_NET_CTRL_GUEST_OFFLOADS => { - let features = mem - .read_obj::(data_desc.addr) + let features = desc_chain + .memory() + .read_obj::(data_desc.addr()) .map_err(Error::GuestMemory)?; if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET { warn!("Unsupported command: {}", ctrl_hdr.cmd); @@ -102,17 +115,24 @@ impl CtrlQueue { } }; - mem.write_obj( - if ok { VIRTIO_NET_OK } else { VIRTIO_NET_ERR } as u8, - status_desc.addr, - ) - .map_err(Error::GuestMemory)?; - used_desc_heads.push((avail_desc.index, avail_desc.len)); + desc_chain + .memory() + .write_obj( + if ok { VIRTIO_NET_OK } else { VIRTIO_NET_ERR } as u8, + status_desc.addr(), + ) + .map_err(Error::GuestMemory)?; + let len = ctrl_desc.len() + data_desc.len() + status_desc.len(); + used_desc_heads.push((desc_chain.head_index(), len)); } for (desc_index, len) in used_desc_heads.iter() { - queue.add_used(mem, *desc_index, *len); - queue.update_avail_event(mem); + queue + .add_used(*desc_index, *len) + .map_err(Error::QueueAddUsed)?; + queue + .enable_notification() + .map_err(Error::QueueEnableNotification)?; } Ok(!used_desc_heads.is_empty()) diff --git a/net_util/src/queue_pair.rs b/net_util/src/queue_pair.rs index 59134205d..ecefab3f9 100644 --- a/net_util/src/queue_pair.rs +++ b/net_util/src/queue_pair.rs @@ -10,8 +10,8 @@ use std::num::Wrapping; use std::os::unix::io::{AsRawFd, RawFd}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use vm_memory::{Bytes, GuestAddressSpace, GuestMemory, GuestMemoryAtomic}; -use vm_virtio::Queue; +use virtio_queue::Queue; +use vm_memory::{Bytes, GuestMemory, GuestMemoryAtomic}; #[derive(Clone)] pub struct TxVirtio { @@ -35,78 +35,93 @@ impl TxVirtio { pub fn process_desc_chain( &mut self, - mem: &GuestMemoryMmap, tap: &mut Tap, - queue: &mut Queue, + queue: &mut Queue>, rate_limiter: &mut Option, ) -> Result { let mut retry_write = false; let mut rate_limit_reached = false; - while let Some(avail_desc) = queue.iter(mem).next() { - if rate_limit_reached { - queue.go_to_previous_position(); + + loop { + let used_desc_head: (u16, u32); + let mut avail_iter = queue + .iter() + .map_err(NetQueuePairError::QueueIteratorFailed)?; + + if let Some(mut desc_chain) = avail_iter.next() { + if rate_limit_reached { + avail_iter.go_to_previous_position(); + break; + } + + let mut next_desc = desc_chain.next(); + + let mut iovecs = Vec::new(); + while let Some(desc) = next_desc { + if !desc.is_write_only() && desc.len() > 0 { + let buf = desc_chain + .memory() + .get_slice(desc.addr(), desc.len() as usize) + .map_err(NetQueuePairError::GuestMemory)? + .as_ptr(); + let iovec = libc::iovec { + iov_base: buf as *mut libc::c_void, + iov_len: desc.len() as libc::size_t, + }; + iovecs.push(iovec); + } + next_desc = desc_chain.next(); + } + + let len = if !iovecs.is_empty() { + let result = unsafe { + libc::writev( + tap.as_raw_fd() as libc::c_int, + iovecs.as_ptr() as *const libc::iovec, + iovecs.len() as libc::c_int, + ) + }; + + if result < 0 { + let e = std::io::Error::last_os_error(); + + /* EAGAIN */ + if e.kind() == std::io::ErrorKind::WouldBlock { + avail_iter.go_to_previous_position(); + retry_write = true; + break; + } + error!("net: tx: failed writing to tap: {}", e); + return Err(NetQueuePairError::WriteTap(e)); + } + + self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64); + self.counter_frames += Wrapping(1); + + result as u32 + } else { + 0 + }; + + used_desc_head = (desc_chain.head_index(), len); + + // For the sake of simplicity (similar to the RX rate limiting), we always + // let the 'last' descriptor chain go-through even if it was over the rate + // limit, and simply stop processing oncoming `avail_desc` if any. + if let Some(rate_limiter) = rate_limiter { + rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops) + || !rate_limiter.consume(len as u64, TokenType::Bytes); + } + } else { break; } - let head_index = avail_desc.index; - let mut next_desc = Some(avail_desc); - - let mut iovecs = Vec::new(); - while let Some(desc) = next_desc { - if !desc.is_write_only() && desc.len > 0 { - let buf = mem - .get_slice(desc.addr, desc.len as usize) - .map_err(NetQueuePairError::GuestMemory)? - .as_ptr(); - let iovec = libc::iovec { - iov_base: buf as *mut libc::c_void, - iov_len: desc.len as libc::size_t, - }; - iovecs.push(iovec); - } - next_desc = desc.next_descriptor(); - } - - let len = if !iovecs.is_empty() { - let result = unsafe { - libc::writev( - tap.as_raw_fd() as libc::c_int, - iovecs.as_ptr() as *const libc::iovec, - iovecs.len() as libc::c_int, - ) - }; - - if result < 0 { - let e = std::io::Error::last_os_error(); - - /* EAGAIN */ - if e.kind() == std::io::ErrorKind::WouldBlock { - queue.go_to_previous_position(); - retry_write = true; - break; - } - error!("net: tx: failed writing to tap: {}", e); - return Err(NetQueuePairError::WriteTap(e)); - } - - self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64); - self.counter_frames += Wrapping(1); - - result as u32 - } else { - 0 - }; - - queue.add_used(mem, head_index, 0); - queue.update_avail_event(mem); - - // For the sake of simplicity (similar to the RX rate limiting), we always - // let the 'last' descriptor chain go-through even if it was over the rate - // limit, and simply stop processing oncoming `avail_desc` if any. - if let Some(rate_limiter) = rate_limiter { - rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops) - || !rate_limiter.consume(len as u64, TokenType::Bytes); - } + queue + .add_used(used_desc_head.0, used_desc_head.1) + .map_err(NetQueuePairError::QueueAddUsed)?; + queue + .enable_notification() + .map_err(NetQueuePairError::QueueEnableNotification)?; } Ok(retry_write) @@ -135,87 +150,106 @@ impl RxVirtio { pub fn process_desc_chain( &mut self, - mem: &GuestMemoryMmap, tap: &mut Tap, - queue: &mut Queue, + queue: &mut Queue>, rate_limiter: &mut Option, ) -> Result { let mut exhausted_descs = true; let mut rate_limit_reached = false; - while let Some(avail_desc) = queue.iter(mem).next() { - if rate_limit_reached { - exhausted_descs = false; - queue.go_to_previous_position(); + loop { + let used_desc_head: (u16, u32); + let mut avail_iter = queue + .iter() + .map_err(NetQueuePairError::QueueIteratorFailed)?; + + if let Some(mut desc_chain) = avail_iter.next() { + if rate_limit_reached { + exhausted_descs = false; + avail_iter.go_to_previous_position(); + break; + } + + let desc = desc_chain + .next() + .ok_or(NetQueuePairError::DescriptorChainTooShort)?; + let num_buffers_addr = desc_chain.memory().checked_offset(desc.addr(), 10).unwrap(); + let mut next_desc = Some(desc); + + let mut iovecs = Vec::new(); + while let Some(desc) = next_desc { + if desc.is_write_only() && desc.len() > 0 { + let buf = desc_chain + .memory() + .get_slice(desc.addr(), desc.len() as usize) + .map_err(NetQueuePairError::GuestMemory)? + .as_ptr(); + let iovec = libc::iovec { + iov_base: buf as *mut libc::c_void, + iov_len: desc.len() as libc::size_t, + }; + iovecs.push(iovec); + } + next_desc = desc_chain.next(); + } + + let len = if !iovecs.is_empty() { + let result = unsafe { + libc::readv( + tap.as_raw_fd() as libc::c_int, + iovecs.as_ptr() as *const libc::iovec, + iovecs.len() as libc::c_int, + ) + }; + if result < 0 { + let e = std::io::Error::last_os_error(); + exhausted_descs = false; + avail_iter.go_to_previous_position(); + + /* EAGAIN */ + if e.kind() == std::io::ErrorKind::WouldBlock { + break; + } + + error!("net: rx: failed reading from tap: {}", e); + return Err(NetQueuePairError::ReadTap(e)); + } + + // Write num_buffers to guest memory. We simply write 1 as we + // never spread the frame over more than one descriptor chain. + desc_chain + .memory() + .write_obj(1u16, num_buffers_addr) + .map_err(NetQueuePairError::GuestMemory)?; + + self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64); + self.counter_frames += Wrapping(1); + + result as u32 + } else { + 0 + }; + + used_desc_head = (desc_chain.head_index(), len); + + // For the sake of simplicity (keeping the handling of RX_QUEUE_EVENT and + // RX_TAP_EVENT totally asynchronous), we always let the 'last' descriptor + // chain go-through even if it was over the rate limit, and simply stop + // processing oncoming `avail_desc` if any. + if let Some(rate_limiter) = rate_limiter { + rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops) + || !rate_limiter.consume(len as u64, TokenType::Bytes); + } + } else { break; } - let head_index = avail_desc.index; - let num_buffers_addr = mem.checked_offset(avail_desc.addr, 10).unwrap(); - let mut next_desc = Some(avail_desc); - - let mut iovecs = Vec::new(); - while let Some(desc) = next_desc { - if desc.is_write_only() && desc.len > 0 { - let buf = mem - .get_slice(desc.addr, desc.len as usize) - .map_err(NetQueuePairError::GuestMemory)? - .as_ptr(); - let iovec = libc::iovec { - iov_base: buf as *mut libc::c_void, - iov_len: desc.len as libc::size_t, - }; - iovecs.push(iovec); - } - next_desc = desc.next_descriptor(); - } - - let len = if !iovecs.is_empty() { - let result = unsafe { - libc::readv( - tap.as_raw_fd() as libc::c_int, - iovecs.as_ptr() as *const libc::iovec, - iovecs.len() as libc::c_int, - ) - }; - if result < 0 { - let e = std::io::Error::last_os_error(); - exhausted_descs = false; - queue.go_to_previous_position(); - - /* EAGAIN */ - if e.kind() == std::io::ErrorKind::WouldBlock { - break; - } - - error!("net: rx: failed reading from tap: {}", e); - return Err(NetQueuePairError::ReadTap(e)); - } - - // Write num_buffers to guest memory. We simply write 1 as we - // never spread the frame over more than one descriptor chain. - mem.write_obj(1u16, num_buffers_addr) - .map_err(NetQueuePairError::GuestMemory)?; - - self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64); - self.counter_frames += Wrapping(1); - - result as u32 - } else { - 0 - }; - - queue.add_used(mem, head_index, len); - queue.update_avail_event(mem); - - // For the sake of simplicity (keeping the handling of RX_QUEUE_EVENT and - // RX_TAP_EVENT totally asynchronous), we always let the 'last' descriptor - // chain go-through even if it was over the rate limit, and simply stop - // processing oncoming `avail_desc` if any. - if let Some(rate_limiter) = rate_limiter { - rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops) - || !rate_limiter.consume(len as u64, TokenType::Bytes); - } + queue + .add_used(used_desc_head.0, used_desc_head.1) + .map_err(NetQueuePairError::QueueAddUsed)?; + queue + .enable_notification() + .map_err(NetQueuePairError::QueueEnableNotification)?; } Ok(exhausted_descs) @@ -244,10 +278,19 @@ pub enum NetQueuePairError { ReadTap(io::Error), /// Error related to guest memory GuestMemory(vm_memory::GuestMemoryError), + /// Returned an error while iterating through the queue + QueueIteratorFailed(virtio_queue::Error), + /// Descriptor chain is too short + DescriptorChainTooShort, + /// Failed to determine if queue needed notification + QueueNeedsNotification(virtio_queue::Error), + /// Failed to enable notification on the queue + QueueEnableNotification(virtio_queue::Error), + /// Failed to add used index to the queue + QueueAddUsed(virtio_queue::Error), } pub struct NetQueuePair { - pub mem: Option>, pub tap: Tap, // With epoll each FD must be unique. So in order to filter the // events we need to get a second FD responding to the original @@ -268,16 +311,13 @@ pub struct NetQueuePair { } impl NetQueuePair { - pub fn process_tx(&mut self, queue: &mut Queue) -> Result { - let mem = self - .mem - .as_ref() - .ok_or(NetQueuePairError::NoMemoryConfigured) - .map(|m| m.memory())?; - + pub fn process_tx( + &mut self, + queue: &mut Queue>, + ) -> Result { let tx_tap_retry = self.tx - .process_desc_chain(&mem, &mut self.tap, queue, &mut self.tx_rate_limiter)?; + .process_desc_chain(&mut self.tap, queue, &mut self.tx_rate_limiter)?; // We got told to try again when writing to the tap. Wait for the TAP to be writable if tx_tap_retry && !self.tx_tap_listening { @@ -311,20 +351,19 @@ impl NetQueuePair { self.tx.counter_bytes = Wrapping(0); self.tx.counter_frames = Wrapping(0); - Ok(queue.needs_notification(&mem, queue.next_used)) + queue + .needs_notification() + .map_err(NetQueuePairError::QueueNeedsNotification) } - pub fn process_rx(&mut self, queue: &mut Queue) -> Result { - let mem = self - .mem - .as_ref() - .ok_or(NetQueuePairError::NoMemoryConfigured) - .map(|m| m.memory())?; - + pub fn process_rx( + &mut self, + queue: &mut Queue>, + ) -> Result { self.rx_desc_avail = !self .rx - .process_desc_chain(&mem, &mut self.tap, queue, &mut self.rx_rate_limiter)?; + .process_desc_chain(&mut self.tap, queue, &mut self.rx_rate_limiter)?; let rate_limit_reached = self .rx_rate_limiter .as_ref() @@ -353,6 +392,8 @@ impl NetQueuePair { self.rx.counter_bytes = Wrapping(0); self.rx.counter_frames = Wrapping(0); - Ok(queue.needs_notification(&mem, queue.next_used)) + queue + .needs_notification() + .map_err(NetQueuePairError::QueueNeedsNotification) } } diff --git a/vhost_user_backend/Cargo.toml b/vhost_user_backend/Cargo.toml index a02f76b6c..57bc2fb4f 100644 --- a/vhost_user_backend/Cargo.toml +++ b/vhost_user_backend/Cargo.toml @@ -12,6 +12,7 @@ epoll = "4.3.1" libc = "0.2.104" log = "0.4.14" virtio-bindings = "0.1.0" +virtio-queue = { path = "../virtio-queue" } vm-memory = { version = "0.6.0", features = ["backend-bitmap"] } vm-virtio = { path = "../vm-virtio" } vmm-sys-util = "0.9.0" diff --git a/vhost_user_backend/src/lib.rs b/vhost_user_backend/src/lib.rs index ec9e78ad5..f0e6febee 100644 --- a/vhost_user_backend/src/lib.rs +++ b/vhost_user_backend/src/lib.rs @@ -10,7 +10,6 @@ extern crate log; use std::error; use std::fs::File; use std::io; -use std::num::Wrapping; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::result; use std::sync::{Arc, Mutex, RwLock}; @@ -26,9 +25,10 @@ use vhost::vhost_user::{ VhostUserSlaveReqHandlerMut, }; use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX; +use virtio_queue::Queue; use vm_memory::guest_memory::FileOffset; -use vm_memory::{bitmap::AtomicBitmap, GuestAddress, MmapRegion}; -use vm_virtio::Queue; +use vm_memory::GuestAddressSpace; +use vm_memory::{bitmap::AtomicBitmap, GuestAddress, GuestMemoryAtomic, MmapRegion}; use vmm_sys_util::eventfd::EventFd; pub type GuestMemoryMmap = vm_memory::GuestMemoryMmap; @@ -83,9 +83,6 @@ pub trait VhostUserBackend: Send + Sync + 'static { /// Tell the backend if EVENT_IDX has been negotiated. fn set_event_idx(&mut self, enabled: bool); - /// Update guest memory regions. - fn update_memory(&mut self, mem: GuestMemoryMmap) -> result::Result<(), io::Error>; - /// This function gets called if the backend registered some additional /// listeners onto specific file descriptors. The library can handle /// virtqueues on its own, but does not know what to do with events @@ -232,7 +229,7 @@ struct AddrMapping { } pub struct Vring { - queue: Queue, + queue: Queue>, kick: Option, call: Option, #[allow(dead_code)] @@ -241,9 +238,9 @@ pub struct Vring { } impl Vring { - fn new(max_queue_size: u16) -> Self { + fn new(mem: GuestMemoryAtomic, max_queue_size: u16) -> Self { Vring { - queue: Queue::new(max_queue_size), + queue: Queue::new(mem, max_queue_size), kick: None, call: None, err: None, @@ -251,7 +248,7 @@ impl Vring { } } - pub fn mut_queue(&mut self) -> &mut Queue { + pub fn mut_queue(&mut self) -> &mut Queue> { &mut self.queue } @@ -468,7 +465,7 @@ struct VhostUserHandler { max_queue_size: usize, queues_per_thread: Vec, mappings: Vec, - guest_memory: Option, + guest_memory: GuestMemoryAtomic, vrings: Vec>>, worker_threads: Vec>>, } @@ -478,10 +475,14 @@ impl VhostUserHandler { let num_queues = backend.read().unwrap().num_queues(); let max_queue_size = backend.read().unwrap().max_queue_size(); let queues_per_thread = backend.read().unwrap().queues_per_thread(); + let guest_memory = GuestMemoryAtomic::new(GuestMemoryMmap::new()); let mut vrings: Vec>> = Vec::new(); for _ in 0..num_queues { - let vring = Arc::new(RwLock::new(Vring::new(max_queue_size as u16))); + let vring = Arc::new(RwLock::new(Vring::new( + guest_memory.clone(), + max_queue_size as u16, + ))); vrings.push(vring); } @@ -544,7 +545,7 @@ impl VhostUserHandler { max_queue_size, queues_per_thread, mappings: Vec::new(), - guest_memory: None, + guest_memory, vrings, worker_threads, }) @@ -649,15 +650,7 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { let mem = GuestMemoryMmap::from_ranges_with_files(regions).map_err(|e| { VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e)) })?; - self.backend - .write() - .unwrap() - .update_memory(mem.clone()) - .map_err(|e| { - VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e)) - })?; - - self.guest_memory = Some(mem); + self.guest_memory.lock().unwrap().replace(mem); self.mappings = mappings; Ok(()) @@ -671,7 +664,12 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { if index as usize >= self.num_queues || num == 0 || num as usize > self.max_queue_size { return Err(VhostUserError::InvalidParam); } - self.vrings[index as usize].write().unwrap().queue.size = num as u16; + self.vrings[index as usize] + .write() + .unwrap() + .queue + .state + .size = num as u16; Ok(()) } @@ -702,13 +700,20 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { .write() .unwrap() .queue + .state .desc_table = GuestAddress(desc_table); self.vrings[index as usize] .write() .unwrap() .queue + .state .avail_ring = GuestAddress(avail_ring); - self.vrings[index as usize].write().unwrap().queue.used_ring = GuestAddress(used_ring); + self.vrings[index as usize] + .write() + .unwrap() + .queue + .state + .used_ring = GuestAddress(used_ring); Ok(()) } else { Err(VhostUserError::InvalidParam) @@ -720,8 +725,7 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { .write() .unwrap() .queue - .next_avail = Wrapping(base as u16); - self.vrings[index as usize].write().unwrap().queue.next_used = Wrapping(base as u16); + .set_next_avail(base as u16); let event_idx: bool = (self.acked_features & (1 << VIRTIO_RING_F_EVENT_IDX)) != 0; self.vrings[index as usize] @@ -742,7 +746,12 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { // that file descriptor is readable) on the descriptor specified by // VHOST_USER_SET_VRING_KICK, and stop ring upon receiving // VHOST_USER_GET_VRING_BASE. - self.vrings[index as usize].write().unwrap().queue.ready = false; + self.vrings[index as usize] + .write() + .unwrap() + .queue + .state + .ready = false; if let Some(fd) = self.vrings[index as usize].read().unwrap().kick.as_ref() { for (thread_index, queues_mask) in self.queues_per_thread.iter().enumerate() { let shifted_queues_mask = queues_mask >> index; @@ -764,8 +773,7 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { .read() .unwrap() .queue - .next_avail - .0 as u16; + .next_avail(); Ok(VhostUserVringState::new(index, u32::from(next_avail))) } @@ -783,7 +791,12 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { // that file descriptor is readable) on the descriptor specified by // VHOST_USER_SET_VRING_KICK, and stop ring upon receiving // VHOST_USER_GET_VRING_BASE. - self.vrings[index as usize].write().unwrap().queue.ready = true; + self.vrings[index as usize] + .write() + .unwrap() + .queue + .state + .ready = true; if let Some(fd) = self.vrings[index as usize].read().unwrap().kick.as_ref() { for (thread_index, queues_mask) in self.queues_per_thread.iter().enumerate() { let shifted_queues_mask = queues_mask >> index; @@ -890,25 +903,14 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { )?, ); - let guest_memory = if let Some(guest_memory) = &self.guest_memory { - guest_memory.insert_region(guest_region).map_err(|e| { - VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e)) - })? - } else { - GuestMemoryMmap::from_arc_regions(vec![guest_region]).map_err(|e| { - VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e)) - })? - }; - - self.backend - .write() - .unwrap() - .update_memory(guest_memory.clone()) + let guest_memory = self + .guest_memory + .memory() + .insert_region(guest_region) .map_err(|e| { VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e)) })?; - - self.guest_memory = Some(guest_memory); + self.guest_memory.lock().unwrap().replace(guest_memory); self.mappings.push(AddrMapping { vmm_addr: region.user_addr, @@ -920,26 +922,14 @@ impl VhostUserSlaveReqHandlerMut for VhostUserHandler { } fn remove_mem_region(&mut self, region: &VhostUserSingleMemoryRegion) -> VhostUserResult<()> { - let guest_memory = if let Some(guest_memory) = &self.guest_memory { - let (updated_guest_memory, _) = guest_memory - .remove_region(GuestAddress(region.guest_phys_addr), region.memory_size) - .map_err(|e| { - VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e)) - })?; - updated_guest_memory - } else { - return Err(VhostUserError::InvalidOperation); - }; - - self.backend - .write() - .unwrap() - .update_memory(guest_memory.clone()) + let (guest_memory, _) = self + .guest_memory + .memory() + .remove_region(GuestAddress(region.guest_phys_addr), region.memory_size) .map_err(|e| { VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e)) })?; - - self.guest_memory = Some(guest_memory); + self.guest_memory.lock().unwrap().replace(guest_memory); self.mappings .retain(|mapping| mapping.gpa_base != region.guest_phys_addr); diff --git a/vhost_user_block/src/lib.rs b/vhost_user_block/src/lib.rs index eed25c1de..d821d22da 100644 --- a/vhost_user_block/src/lib.rs +++ b/vhost_user_block/src/lib.rs @@ -17,7 +17,6 @@ use std::fs::File; use std::fs::OpenOptions; use std::io::Read; use std::io::{Seek, SeekFrom, Write}; -use std::num::Wrapping; use std::ops::DerefMut; use std::os::unix::fs::OpenOptionsExt; use std::path::PathBuf; @@ -30,7 +29,7 @@ use std::vec::Vec; use std::{convert, error, fmt, io}; use vhost::vhost_user::message::*; use vhost::vhost_user::Listener; -use vhost_user_backend::{GuestMemoryMmap, VhostUserBackend, VhostUserDaemon, Vring}; +use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring}; use virtio_bindings::bindings::virtio_blk::*; use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX; use vm_memory::ByteValued; @@ -87,7 +86,6 @@ impl convert::From for io::Error { } struct VhostUserBlkThread { - mem: Option, disk_image: Arc>, disk_image_id: Vec, disk_nsectors: u64, @@ -104,7 +102,6 @@ impl VhostUserBlkThread { writeback: Arc, ) -> Result { Ok(VhostUserBlkThread { - mem: None, disk_image, disk_image_id, disk_nsectors, @@ -116,22 +113,18 @@ impl VhostUserBlkThread { fn process_queue(&mut self, vring: &mut Vring) -> bool { let mut used_any = false; - let mem = match self.mem.as_ref() { - Some(m) => m, - None => return false, - }; - while let Some(head) = vring.mut_queue().iter(mem).next() { + while let Some(mut desc_chain) = vring.mut_queue().iter().unwrap().next() { debug!("got an element in the queue"); let len; - match Request::parse(&head, mem) { + match Request::parse(&mut desc_chain) { Ok(mut request) => { debug!("element is a valid request"); request.set_writeback(self.writeback.load(Ordering::Acquire)); let status = match request.execute( &mut self.disk_image.lock().unwrap().deref_mut(), self.disk_nsectors, - mem, + desc_chain.memory(), &self.disk_image_id, ) { Ok(l) => { @@ -143,7 +136,10 @@ impl VhostUserBlkThread { e.status() } }; - mem.write_obj(status, request.status_addr).unwrap(); + desc_chain + .memory() + .write_obj(status, request.status_addr) + .unwrap(); } Err(err) => { error!("failed to parse available descriptor chain: {:?}", err); @@ -153,8 +149,8 @@ impl VhostUserBlkThread { if self.event_idx { let queue = vring.mut_queue(); - if let Some(used_idx) = queue.add_used(mem, head.index, len) { - if queue.needs_notification(mem, Wrapping(used_idx)) { + if queue.add_used(desc_chain.head_index(), len).is_ok() { + if queue.needs_notification().unwrap() { debug!("signalling queue"); vring.signal_used_queue().unwrap(); } else { @@ -164,7 +160,10 @@ impl VhostUserBlkThread { } } else { debug!("signalling queue"); - vring.mut_queue().add_used(mem, head.index, len); + vring + .mut_queue() + .add_used(desc_chain.head_index(), len) + .unwrap(); vring.signal_used_queue().unwrap(); used_any = true; } @@ -316,13 +315,6 @@ impl VhostUserBackend for VhostUserBlkBackend { } } - fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> { - for thread in self.threads.iter() { - thread.lock().unwrap().mem = Some(mem.clone()); - } - Ok(()) - } - fn handle_event( &self, device_event: u16, @@ -360,9 +352,7 @@ impl VhostUserBackend for VhostUserBlkBackend { // calling process_queue() until it stops finding new // requests on the queue. loop { - vring - .mut_queue() - .update_avail_event(thread.mem.as_ref().unwrap()); + vring.mut_queue().enable_notification().unwrap(); if !thread.process_queue(&mut vring) { break; } diff --git a/vhost_user_net/src/lib.rs b/vhost_user_net/src/lib.rs index e0cd34632..25355d485 100644 --- a/vhost_user_net/src/lib.rs +++ b/vhost_user_net/src/lib.rs @@ -22,9 +22,8 @@ use std::sync::{Arc, Mutex, RwLock}; use std::vec::Vec; use vhost::vhost_user::message::*; use vhost::vhost_user::Listener; -use vhost_user_backend::{GuestMemoryMmap, VhostUserBackend, VhostUserDaemon, Vring, VringWorker}; +use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker}; use virtio_bindings::bindings::virtio_net::*; -use vm_memory::GuestMemoryAtomic; use vmm_sys_util::eventfd::EventFd; pub type Result = std::result::Result; @@ -81,7 +80,6 @@ impl VhostUserNetThread { Ok(VhostUserNetThread { kill_evt: EventFd::new(EFD_NONBLOCK).map_err(Error::CreateKillEventFd)?, net: NetQueuePair { - mem: None, tap_for_write_epoll: tap.clone(), tap, rx: RxVirtio::new(), @@ -184,13 +182,6 @@ impl VhostUserBackend for VhostUserNetBackend { fn set_event_idx(&mut self, _enabled: bool) {} - fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> { - for thread in self.threads.iter() { - thread.lock().unwrap().net.mem = Some(GuestMemoryAtomic::new(mem.clone())); - } - Ok(()) - } - fn handle_event( &self, device_event: u16, diff --git a/virtio-devices/Cargo.toml b/virtio-devices/Cargo.toml index 0cc62dc01..bddb8afba 100644 --- a/virtio-devices/Cargo.toml +++ b/virtio-devices/Cargo.toml @@ -31,6 +31,7 @@ versionize = "0.1.6" versionize_derive = "0.1.4" vhost = { version = "0.2.0", features = ["vhost-user-master", "vhost-user-slave", "vhost-kern"] } virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] } +virtio-queue = { path = "../virtio-queue" } vm-allocator = { path = "../vm-allocator" } vm-device = { path = "../vm-device" } vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] } diff --git a/virtio-devices/src/balloon.rs b/virtio-devices/src/balloon.rs index 5ebd7b632..d92201e49 100644 --- a/virtio-devices/src/balloon.rs +++ b/virtio-devices/src/balloon.rs @@ -13,8 +13,8 @@ // limitations under the License. use super::{ - ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, - VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, + ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, + VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, }; use crate::seccomp_filters::Thread; use crate::thread_helper::spawn_virtio_thread; @@ -31,11 +31,9 @@ use std::sync::mpsc; use std::sync::{Arc, Barrier, Mutex}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; +use virtio_queue::Queue; use vm_memory::GuestMemory; -use vm_memory::{ - Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, - GuestMemoryError, -}; +use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryAtomic, GuestMemoryError}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; use vmm_sys_util::eventfd::EventFd; @@ -81,6 +79,12 @@ pub enum Error { ProcessQueueWrongEvType(u16), // Fail tp signal FailedSignal(io::Error), + /// Descriptor chain is too short + DescriptorChainTooShort, + /// Failed adding used index + QueueAddUsed(virtio_queue::Error), + /// Failed creating an iterator over the queue + QueueIterator(virtio_queue::Error), } // Got from include/uapi/linux/virtio_balloon.h @@ -152,8 +156,7 @@ impl VirtioBalloonResize { struct BalloonEpollHandler { config: Arc>, resize_receiver: VirtioBalloonResizeReceiver, - queues: Vec, - mem: GuestMemoryAtomic, + queues: Vec>>, interrupt_cb: Arc, inflate_queue_evt: EventFd, deflate_queue_evt: EventFd, @@ -165,7 +168,7 @@ impl BalloonEpollHandler { fn signal( &self, int_type: &VirtioInterruptType, - queue: Option<&Queue>, + queue: Option<&Queue>>, ) -> result::Result<(), Error> { self.interrupt_cb.trigger(int_type, queue).map_err(|e| { error!("Failed to signal used queue: {:?}", e); @@ -182,38 +185,45 @@ impl BalloonEpollHandler { let mut used_desc_heads = [0; QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in self.queues[queue_index].iter(&mem) { - used_desc_heads[used_count] = avail_desc.index; + for mut desc_chain in self.queues[queue_index] + .iter() + .map_err(Error::QueueIterator)? + { + let desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?; + + used_desc_heads[used_count] = desc_chain.head_index(); used_count += 1; let data_chunk_size = size_of::(); // The head contains the request type which MUST be readable. - if avail_desc.is_write_only() { + if desc.is_write_only() { error!("The head contains the request type is not right"); return Err(Error::UnexpectedWriteOnlyDescriptor); } - if avail_desc.len as usize % data_chunk_size != 0 { - error!("the request size {} is not right", avail_desc.len); + if desc.len() as usize % data_chunk_size != 0 { + error!("the request size {} is not right", desc.len()); return Err(Error::InvalidRequest); } let mut offset = 0u64; - while offset < avail_desc.len as u64 { - let addr = avail_desc.addr.checked_add(offset).unwrap(); - let pfn: u32 = mem.read_obj(addr).map_err(Error::GuestMemory)?; + while offset < desc.len() as u64 { + let addr = desc.addr().checked_add(offset).unwrap(); + let pfn: u32 = desc_chain + .memory() + .read_obj(addr) + .map_err(Error::GuestMemory)?; offset += data_chunk_size as u64; let gpa = (pfn as u64) << VIRTIO_BALLOON_PFN_SHIFT; - if let Ok(hva) = mem.get_host_address(GuestAddress(gpa)) { + if let Ok(hva) = desc_chain.memory().get_host_address(GuestAddress(gpa)) { let advice = match ev_type { INFLATE_QUEUE_EVENT => { - let region = - mem.find_region(GuestAddress(gpa)) - .ok_or(Error::GuestMemory( - GuestMemoryError::InvalidGuestAddress(GuestAddress(gpa)), - ))?; + let region = desc_chain.memory().find_region(GuestAddress(gpa)).ok_or( + Error::GuestMemory(GuestMemoryError::InvalidGuestAddress( + GuestAddress(gpa), + )), + )?; if let Some(f_off) = region.file_offset() { let offset = hva as usize - region.as_ptr() as usize; let res = unsafe { @@ -253,7 +263,9 @@ impl BalloonEpollHandler { } for &desc_index in &used_desc_heads[..used_count] { - self.queues[queue_index].add_used(&mem, desc_index, 0); + self.queues[queue_index] + .add_used(desc_index, 0) + .map_err(Error::QueueAddUsed)?; } if used_count > 0 { self.signal(&VirtioInterruptType::Queue, Some(&self.queues[queue_index]))?; @@ -463,9 +475,9 @@ impl VirtioDevice for Balloon { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -478,7 +490,6 @@ impl VirtioDevice for Balloon { ActivateError::BadActivate })?, queues, - mem, interrupt_cb, inflate_queue_evt: queue_evts.remove(0), deflate_queue_evt: queue_evts.remove(0), diff --git a/virtio-devices/src/block.rs b/virtio-devices/src/block.rs index 784eb38a6..1975aa611 100644 --- a/virtio-devices/src/block.rs +++ b/virtio-devices/src/block.rs @@ -10,7 +10,7 @@ use super::Error as DeviceError; use super::{ - ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, + ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST, }; @@ -35,6 +35,7 @@ use std::{collections::HashMap, convert::TryInto}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; use virtio_bindings::bindings::virtio_blk::*; +use virtio_queue::Queue; use vm_memory::{ByteValued, Bytes, GuestAddressSpace, GuestMemoryAtomic}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; @@ -62,6 +63,10 @@ pub enum Error { AsyncRequestFailure, /// Failed synchronizing the file Fsync(AsyncIoError), + /// Failed adding used index + QueueAddUsed(virtio_queue::Error), + /// Failed creating an iterator over the queue + QueueIterator(virtio_queue::Error), } pub type Result = result::Result; @@ -75,7 +80,7 @@ pub struct BlockCounters { } struct BlockEpollHandler { - queue: Queue, + queue: Queue>, mem: GuestMemoryAtomic, disk_image: Box, disk_nsectors: u64, @@ -93,13 +98,13 @@ struct BlockEpollHandler { impl BlockEpollHandler { fn process_queue_submit(&mut self) -> Result { let queue = &mut self.queue; - let mem = self.mem.memory(); let mut used_desc_heads = Vec::new(); let mut used_count = 0; - for avail_desc in queue.iter(&mem) { - let mut request = Request::parse(&avail_desc, &mem).map_err(Error::RequestParsing)?; + let mut avail_iter = queue.iter().map_err(Error::QueueIterator)?; + for mut desc_chain in &mut avail_iter { + let mut request = Request::parse(&mut desc_chain).map_err(Error::RequestParsing)?; if let Some(rate_limiter) = &mut self.rate_limiter { // If limiter.consume() fails it means there is no more TokenType::Ops @@ -107,7 +112,7 @@ impl BlockEpollHandler { if !rate_limiter.consume(1, TokenType::Ops) { // Stop processing the queue and return this descriptor chain to the // avail ring, for later processing. - queue.go_to_previous_position(); + avail_iter.go_to_previous_position(); break; } // Exercise the rate limiter only if this request is of data transfer type. @@ -126,7 +131,7 @@ impl BlockEpollHandler { rate_limiter.manual_replenish(1, TokenType::Ops); // Stop processing the queue and return this descriptor chain to the // avail ring, for later processing. - queue.go_to_previous_position(); + avail_iter.go_to_previous_position(); break; } }; @@ -136,29 +141,34 @@ impl BlockEpollHandler { if request .execute_async( - &mem, + desc_chain.memory(), self.disk_nsectors, self.disk_image.as_mut(), &self.disk_image_id, - avail_desc.index as u64, + desc_chain.head_index() as u64, ) .map_err(Error::RequestExecuting)? { - self.request_list.insert(avail_desc.index, request); + self.request_list.insert(desc_chain.head_index(), request); } else { // We use unwrap because the request parsing process already // checked that the status_addr was valid. - mem.write_obj(VIRTIO_BLK_S_OK, request.status_addr).unwrap(); + desc_chain + .memory() + .write_obj(VIRTIO_BLK_S_OK, request.status_addr) + .unwrap(); // If no asynchronous operation has been submitted, we can // simply return the used descriptor. - used_desc_heads.push((avail_desc.index, 0)); + used_desc_heads.push((desc_chain.head_index(), 0)); used_count += 1; } } for &(desc_index, len) in used_desc_heads.iter() { - queue.add_used(&mem, desc_index, len); + queue + .add_used(desc_index, len) + .map_err(Error::QueueAddUsed)?; } Ok(used_count > 0) @@ -221,7 +231,9 @@ impl BlockEpollHandler { } for &(desc_index, len) in used_desc_heads.iter() { - queue.add_used(&mem, desc_index, len); + queue + .add_used(desc_index, len) + .map_err(Error::QueueAddUsed)?; } self.counters @@ -545,7 +557,7 @@ impl VirtioDevice for Block { &mut self, mem: GuestMemoryAtomic, interrupt_cb: Arc, - mut queues: Vec, + mut queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -557,7 +569,7 @@ impl VirtioDevice for Block { for i in 0..queues.len() { let queue_evt = queue_evts.remove(0); let queue = queues.remove(0); - let queue_size = queue.size; + let queue_size = queue.state.size; let (kill_evt, pause_evt) = self.common.dup_eventfds(); let rate_limiter: Option = self diff --git a/virtio-devices/src/console.rs b/virtio-devices/src/console.rs index 8899c45f9..166da10b8 100644 --- a/virtio-devices/src/console.rs +++ b/virtio-devices/src/console.rs @@ -3,9 +3,9 @@ use super::Error as DeviceError; use super::{ - ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, VirtioCommon, - VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST, - VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1, + ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, VirtioDevice, + VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IOMMU_PLATFORM, + VIRTIO_F_VERSION_1, }; use crate::seccomp_filters::Thread; use crate::thread_helper::spawn_virtio_thread; @@ -24,7 +24,8 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Barrier, Mutex}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; -use vm_memory::{ByteValued, Bytes, GuestAddressSpace, GuestMemoryAtomic}; +use virtio_queue::Queue; +use vm_memory::{ByteValued, Bytes, GuestMemoryAtomic}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; use vmm_sys_util::eventfd::EventFd; @@ -72,8 +73,7 @@ impl Default for VirtioConsoleConfig { unsafe impl ByteValued for VirtioConsoleConfig {} struct ConsoleEpollHandler { - queues: Vec, - mem: GuestMemoryAtomic, + queues: Vec>>, interrupt_cb: Arc, in_buffer: Arc>>, resizer: Arc, @@ -140,17 +140,21 @@ impl ConsoleEpollHandler { return false; } - let mem = self.mem.memory(); - for avail_desc in recv_queue.iter(&mem) { - let len = cmp::min(avail_desc.len as u32, in_buffer.len() as u32); + let mut avail_iter = recv_queue.iter().unwrap(); + for mut desc_chain in &mut avail_iter { + let desc = desc_chain.next().unwrap(); + let len = cmp::min(desc.len() as u32, in_buffer.len() as u32); let source_slice = in_buffer.drain(..len as usize).collect::>(); - if let Err(e) = mem.write_slice(&source_slice[..], avail_desc.addr) { + if let Err(e) = desc_chain + .memory() + .write_slice(&source_slice[..], desc.addr()) + { error!("Failed to write slice: {:?}", e); - recv_queue.go_to_previous_position(); + avail_iter.go_to_previous_position(); break; } - used_desc_heads[used_count] = (avail_desc.index, len); + used_desc_heads[used_count] = (desc_chain.head_index(), len); used_count += 1; if in_buffer.is_empty() { @@ -159,7 +163,7 @@ impl ConsoleEpollHandler { } for &(desc_index, len) in &used_desc_heads[..used_count] { - recv_queue.add_used(&mem, desc_index, len); + recv_queue.add_used(desc_index, len).unwrap(); } used_count > 0 @@ -177,20 +181,20 @@ impl ConsoleEpollHandler { let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in trans_queue.iter(&mem) { - let len; + for mut desc_chain in trans_queue.iter().unwrap() { + let desc = desc_chain.next().unwrap(); if let Some(ref mut out) = self.endpoint.out_file() { - let _ = mem.write_to(avail_desc.addr, out, avail_desc.len as usize); + let _ = desc_chain + .memory() + .write_to(desc.addr(), out, desc.len() as usize); let _ = out.flush(); } - len = avail_desc.len; - used_desc_heads[used_count] = (avail_desc.index, len); + used_desc_heads[used_count] = (desc_chain.head_index(), desc.len()); used_count += 1; } for &(desc_index, len) in &used_desc_heads[..used_count] { - trans_queue.add_used(&mem, desc_index, len); + trans_queue.add_used(desc_index, len).unwrap(); } used_count > 0 } @@ -477,9 +481,9 @@ impl VirtioDevice for Console { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -498,7 +502,6 @@ impl VirtioDevice for Console { let mut handler = ConsoleEpollHandler { queues, - mem, interrupt_cb, in_buffer: self.in_buffer.clone(), endpoint: self.endpoint.clone(), diff --git a/virtio-devices/src/device.rs b/virtio-devices/src/device.rs index d761d1bac..001bf6929 100644 --- a/virtio-devices/src/device.rs +++ b/virtio-devices/src/device.rs @@ -6,7 +6,7 @@ // // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause -use crate::{ActivateError, ActivateResult, Error, Queue}; +use crate::{ActivateError, ActivateResult, Error}; use crate::{GuestMemoryMmap, GuestRegionMmap}; use libc::EFD_NONBLOCK; use std::collections::HashMap; @@ -17,6 +17,7 @@ use std::sync::{ Arc, Barrier, }; use std::thread; +use virtio_queue::Queue; use vm_memory::{GuestAddress, GuestMemoryAtomic, GuestUsize}; use vm_migration::{MigratableError, Pausable}; use vm_virtio::VirtioDeviceType; @@ -31,9 +32,13 @@ pub trait VirtioInterrupt: Send + Sync { fn trigger( &self, int_type: &VirtioInterruptType, - queue: Option<&Queue>, + queue: Option<&Queue>>, ) -> std::result::Result<(), std::io::Error>; - fn notifier(&self, _int_type: &VirtioInterruptType, _queue: Option<&Queue>) -> Option { + fn notifier( + &self, + _int_type: &VirtioInterruptType, + _queue: Option<&Queue>>, + ) -> Option { None } } @@ -107,7 +112,7 @@ pub trait VirtioDevice: Send { &mut self, mem: GuestMemoryAtomic, interrupt_evt: Arc, - queues: Vec, + queues: Vec>>, queue_evts: Vec, ) -> ActivateResult; @@ -247,7 +252,7 @@ impl VirtioCommon { pub fn activate( &mut self, - queues: &[Queue], + queues: &[Queue>], queue_evts: &[EventFd], interrupt_cb: &Arc, ) -> ActivateResult { diff --git a/virtio-devices/src/epoll_helper.rs b/virtio-devices/src/epoll_helper.rs index 4cfcf924f..8dab31916 100644 --- a/virtio-devices/src/epoll_helper.rs +++ b/virtio-devices/src/epoll_helper.rs @@ -26,6 +26,7 @@ pub enum EpollHelperError { Ctl(std::io::Error), IoError(std::io::Error), Wait(std::io::Error), + QueueRingIndex(virtio_queue::Error), } pub const EPOLL_HELPER_EVENT_PAUSE: u16 = 0; diff --git a/virtio-devices/src/iommu.rs b/virtio-devices/src/iommu.rs index ea747f5a6..e795782ac 100644 --- a/virtio-devices/src/iommu.rs +++ b/virtio-devices/src/iommu.rs @@ -4,8 +4,8 @@ use super::Error as DeviceError; use super::{ - ActivateResult, DescriptorChain, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, - VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, + ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, VirtioDevice, + VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, }; use crate::seccomp_filters::Thread; use crate::thread_helper::spawn_virtio_thread; @@ -23,11 +23,9 @@ use std::sync::atomic::AtomicBool; use std::sync::{Arc, Barrier, RwLock}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; +use virtio_queue::{AccessPlatform, DescriptorChain, Queue}; use vm_device::dma_mapping::ExternalDmaMapping; -use vm_memory::{ - Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, - GuestMemoryError, -}; +use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryAtomic, GuestMemoryError}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; use vmm_sys_util::eventfd::EventFd; @@ -352,27 +350,36 @@ impl Request { // is created based on the information provided from the guest driver for // virtio-iommu (giving the link device_id <=> domain). fn parse( - avail_desc: &DescriptorChain, - mem: &GuestMemoryMmap, + desc_chain: &mut DescriptorChain>, mapping: &Arc, ext_mapping: &BTreeMap>, ext_domain_mapping: &mut BTreeMap>, msi_iova_space: (u64, u64), ) -> result::Result { - // The head contains the request type which MUST be readable. - if avail_desc.is_write_only() { + let desc = desc_chain + .next() + .ok_or(Error::DescriptorChainTooShort) + .map_err(|e| { + error!("Missing head descriptor"); + e + })?; + + // The descriptor contains the request type which MUST be readable. + if desc.is_write_only() { return Err(Error::UnexpectedWriteOnlyDescriptor); } - if (avail_desc.len as usize) < size_of::() { + if (desc.len() as usize) < size_of::() { return Err(Error::InvalidRequest); } - let req_head: VirtioIommuReqHead = - mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?; + let req_head: VirtioIommuReqHead = desc_chain + .memory() + .read_obj(desc.addr()) + .map_err(Error::GuestMemory)?; let req_offset = size_of::(); - let desc_size_left = (avail_desc.len as usize) - req_offset; - let req_addr = if let Some(addr) = avail_desc.addr.checked_add(req_offset as u64) { + let desc_size_left = (desc.len() as usize) - req_offset; + let req_addr = if let Some(addr) = desc.addr().checked_add(req_offset as u64) { addr } else { return Err(Error::InvalidRequest); @@ -389,7 +396,8 @@ impl Request { return Err(Error::InvalidAttachRequest); } - let req: VirtioIommuReqAttach = mem + let req: VirtioIommuReqAttach = desc_chain + .memory() .read_obj(req_addr as GuestAddress) .map_err(Error::GuestMemory)?; debug!("Attach request {:?}", req); @@ -419,7 +427,8 @@ impl Request { return Err(Error::InvalidDetachRequest); } - let req: VirtioIommuReqDetach = mem + let req: VirtioIommuReqDetach = desc_chain + .memory() .read_obj(req_addr as GuestAddress) .map_err(Error::GuestMemory)?; debug!("Detach request {:?}", req); @@ -445,7 +454,8 @@ impl Request { return Err(Error::InvalidMapRequest); } - let req: VirtioIommuReqMap = mem + let req: VirtioIommuReqMap = desc_chain + .memory() .read_obj(req_addr as GuestAddress) .map_err(Error::GuestMemory)?; debug!("Map request {:?}", req); @@ -481,7 +491,8 @@ impl Request { return Err(Error::InvalidUnmapRequest); } - let req: VirtioIommuReqUnmap = mem + let req: VirtioIommuReqUnmap = desc_chain + .memory() .read_obj(req_addr as GuestAddress) .map_err(Error::GuestMemory)?; debug!("Unmap request {:?}", req); @@ -510,7 +521,8 @@ impl Request { return Err(Error::InvalidProbeRequest); } - let req: VirtioIommuReqProbe = mem + let req: VirtioIommuReqProbe = desc_chain + .memory() .read_obj(req_addr as GuestAddress) .map_err(Error::GuestMemory)?; debug!("Probe request {:?}", req); @@ -534,16 +546,14 @@ impl Request { _ => return Err(Error::InvalidRequest), }; - let status_desc = avail_desc - .next_descriptor() - .ok_or(Error::DescriptorChainTooShort)?; + let status_desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?; // The status MUST always be writable if !status_desc.is_write_only() { return Err(Error::UnexpectedReadOnlyDescriptor); } - if status_desc.len < hdr_len + size_of::() as u32 { + if status_desc.len() < hdr_len + size_of::() as u32 { return Err(Error::BufferLengthTooSmall); } @@ -553,7 +563,9 @@ impl Request { }; reply.extend_from_slice(tail.as_slice()); - mem.write_slice(reply.as_slice(), status_desc.addr) + desc_chain + .memory() + .write_slice(reply.as_slice(), status_desc.addr()) .map_err(Error::GuestMemory)?; Ok((hdr_len as usize) + size_of::()) @@ -561,8 +573,7 @@ impl Request { } struct IommuEpollHandler { - queues: Vec, - mem: GuestMemoryAtomic, + queues: Vec>>, interrupt_cb: Arc, queue_evts: Vec, kill_evt: EventFd, @@ -577,11 +588,9 @@ impl IommuEpollHandler { fn request_queue(&mut self) -> bool { let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in self.queues[0].iter(&mem) { + for mut desc_chain in self.queues[0].iter().unwrap() { let len = match Request::parse( - &avail_desc, - &mem, + &mut desc_chain, &self.mapping, &self.ext_mapping, &mut self.ext_domain_mapping, @@ -594,12 +603,12 @@ impl IommuEpollHandler { } }; - used_desc_heads[used_count] = (avail_desc.index, len); + used_desc_heads[used_count] = (desc_chain.head_index(), len); used_count += 1; } for &(desc_index, len) in &used_desc_heads[..used_count] { - self.queues[0].add_used(&mem, desc_index, len); + self.queues[0].add_used(desc_index, len).unwrap(); } used_count > 0 } @@ -608,7 +617,10 @@ impl IommuEpollHandler { false } - fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> { + fn signal_used_queue( + &self, + queue: &Queue>, + ) -> result::Result<(), DeviceError> { self.interrupt_cb .trigger(&VirtioInterruptType::Queue, Some(queue)) .map_err(|e| { @@ -666,12 +678,13 @@ impl EpollHelperHandler for IommuEpollHandler { } } -#[derive(Clone, Copy, Versionize)] +#[derive(Clone, Copy, Debug, Versionize)] struct Mapping { gpa: u64, size: u64, } +#[derive(Debug)] pub struct IommuMapping { // Domain related to an endpoint. endpoints: Arc>>, @@ -704,6 +717,24 @@ impl DmaRemapping for IommuMapping { } } +#[derive(Debug)] +pub struct AccessPlatformMapping { + id: u32, + mapping: Arc, +} + +impl AccessPlatformMapping { + pub fn new(id: u32, mapping: Arc) -> Self { + AccessPlatformMapping { id, mapping } + } +} + +impl AccessPlatform for AccessPlatformMapping { + fn translate(&self, base: u64, _size: u64) -> std::result::Result { + self.mapping.translate(self.id, base) + } +} + pub struct Iommu { common: VirtioCommon, id: String, @@ -839,16 +870,15 @@ impl VirtioDevice for Iommu { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; let (kill_evt, pause_evt) = self.common.dup_eventfds(); let mut handler = IommuEpollHandler { queues, - mem, interrupt_cb, queue_evts, kill_evt, diff --git a/virtio-devices/src/lib.rs b/virtio-devices/src/lib.rs index 9e889def5..08608fb89 100644 --- a/virtio-devices/src/lib.rs +++ b/virtio-devices/src/lib.rs @@ -51,7 +51,7 @@ pub use self::rng::*; pub use self::vsock::*; pub use self::watchdog::*; use vm_memory::{bitmap::AtomicBitmap, GuestAddress, GuestMemory}; -use vm_virtio::{queue::*, VirtioDeviceType}; +use vm_virtio::VirtioDeviceType; type GuestMemoryMmap = vm_memory::GuestMemoryMmap; type GuestRegionMmap = vm_memory::GuestRegionMmap; @@ -115,6 +115,8 @@ pub enum Error { SetShmRegionsNotSupported, NetQueuePair(::net_util::NetQueuePairError), ApplySeccompFilter(seccompiler::Error), + QueueAddUsed(virtio_queue::Error), + QueueIterator(virtio_queue::Error), } #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq)] diff --git a/virtio-devices/src/mem.rs b/virtio-devices/src/mem.rs index f59b67a3a..419385815 100644 --- a/virtio-devices/src/mem.rs +++ b/virtio-devices/src/mem.rs @@ -14,9 +14,8 @@ use super::Error as DeviceError; use super::{ - ActivateError, ActivateResult, DescriptorChain, EpollHelper, EpollHelperError, - EpollHelperHandler, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType, - EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, + ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, + VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, }; use crate::seccomp_filters::Thread; use crate::thread_helper::spawn_virtio_thread; @@ -35,10 +34,11 @@ use std::sync::mpsc; use std::sync::{Arc, Barrier, Mutex}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; +use virtio_queue::{DescriptorChain, Queue}; use vm_device::dma_mapping::ExternalDmaMapping; use vm_memory::{ - Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, - GuestMemoryError, GuestMemoryRegion, + Address, ByteValued, Bytes, GuestAddress, GuestMemoryAtomic, GuestMemoryError, + GuestMemoryRegion, }; use vm_migration::protocol::MemoryRangeTable; use vm_migration::{ @@ -277,34 +277,35 @@ struct Request { impl Request { fn parse( - avail_desc: &DescriptorChain, - mem: &GuestMemoryMmap, + desc_chain: &mut DescriptorChain>, ) -> result::Result { - // The head contains the request type which MUST be readable. - if avail_desc.is_write_only() { + let desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?; + // The descriptor contains the request type which MUST be readable. + if desc.is_write_only() { return Err(Error::UnexpectedWriteOnlyDescriptor); } - if avail_desc.len as usize != size_of::() { + if desc.len() as usize != size_of::() { return Err(Error::InvalidRequest); } - let req: VirtioMemReq = mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?; + let req: VirtioMemReq = desc_chain + .memory() + .read_obj(desc.addr()) + .map_err(Error::GuestMemory)?; - let status_desc = avail_desc - .next_descriptor() - .ok_or(Error::DescriptorChainTooShort)?; + let status_desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?; // The status MUST always be writable if !status_desc.is_write_only() { return Err(Error::UnexpectedReadOnlyDescriptor); } - if (status_desc.len as usize) < size_of::() { + if (status_desc.len() as usize) < size_of::() { return Err(Error::BufferLengthTooSmall); } Ok(Request { req, - status_addr: status_desc.addr, + status_addr: status_desc.addr(), }) } @@ -455,8 +456,7 @@ struct MemEpollHandler { blocks_state: Arc>, config: Arc>, resize: ResizeSender, - queue: Queue, - mem: GuestMemoryAtomic, + queue: Queue>, interrupt_cb: Arc, queue_evt: EventFd, kill_evt: EventFd, @@ -656,12 +656,16 @@ impl MemEpollHandler { fn process_queue(&mut self) -> bool { let mut request_list = Vec::new(); let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in self.queue.iter(&mem) { - request_list.push((avail_desc.index, Request::parse(&avail_desc, &mem))); + + for mut desc_chain in self.queue.iter().unwrap() { + request_list.push(( + desc_chain.head_index(), + Request::parse(&mut desc_chain), + desc_chain.memory().clone(), + )); } - for (desc_index, request) in request_list.iter() { + for (head_index, request, memory) in request_list { let len = match request { Err(e) => { error!("failed parse VirtioMemReq: {:?}", e); @@ -671,21 +675,21 @@ impl MemEpollHandler { VIRTIO_MEM_REQ_PLUG => { let resp_type = self.state_change_request(r.req.addr, r.req.nb_blocks, true); - r.send_response(&mem, resp_type, 0u16) + r.send_response(&memory, resp_type, 0u16) } VIRTIO_MEM_REQ_UNPLUG => { let resp_type = self.state_change_request(r.req.addr, r.req.nb_blocks, false); - r.send_response(&mem, resp_type, 0u16) + r.send_response(&memory, resp_type, 0u16) } VIRTIO_MEM_REQ_UNPLUG_ALL => { let resp_type = self.unplug_all(); - r.send_response(&mem, resp_type, 0u16) + r.send_response(&memory, resp_type, 0u16) } VIRTIO_MEM_REQ_STATE => { let (resp_type, resp_state) = self.state_request(r.req.addr, r.req.nb_blocks); - r.send_response(&mem, resp_type, resp_state) + r.send_response(&memory, resp_type, resp_state) } _ => { error!("VirtioMemReq unknown request type {:?}", r.req.req_type); @@ -694,8 +698,7 @@ impl MemEpollHandler { }, }; - self.queue.add_used(&mem, *desc_index, len); - + self.queue.add_used(head_index, len).unwrap(); used_count += 1; } @@ -990,9 +993,9 @@ impl VirtioDevice for Mem { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - mut queues: Vec, + mut queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -1004,7 +1007,6 @@ impl VirtioDevice for Mem { config: self.config.clone(), resize: self.resize.clone(), queue: queues.remove(0), - mem, interrupt_cb, queue_evt: queue_evts.remove(0), kill_evt, diff --git a/virtio-devices/src/net.rs b/virtio-devices/src/net.rs index 68966bbec..96f89c68f 100644 --- a/virtio-devices/src/net.rs +++ b/virtio-devices/src/net.rs @@ -7,7 +7,7 @@ use super::Error as DeviceError; use super::{ - ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, + ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST, }; @@ -35,7 +35,8 @@ use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; use virtio_bindings::bindings::virtio_net::*; use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX; -use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic}; +use virtio_queue::Queue; +use vm_memory::{ByteValued, GuestMemoryAtomic}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; use vmm_sys_util::eventfd::EventFd; @@ -45,12 +46,11 @@ use vmm_sys_util::eventfd::EventFd; const CTRL_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; pub struct NetCtrlEpollHandler { - pub mem: GuestMemoryAtomic, pub kill_evt: EventFd, pub pause_evt: EventFd, pub ctrl_q: CtrlQueue, pub queue_evt: EventFd, - pub queue: Queue, + pub queue: Queue>, } impl NetCtrlEpollHandler { @@ -72,12 +72,11 @@ impl EpollHelperHandler for NetCtrlEpollHandler { let ev_type = event.data as u16; match ev_type { CTRL_QUEUE_EVENT => { - let mem = self.mem.memory(); if let Err(e) = self.queue_evt.read() { error!("failed to get ctl queue event: {:?}", e); return true; } - if let Err(e) = self.ctrl_q.process(&mem, &mut self.queue) { + if let Err(e) = self.ctrl_q.process(&mut self.queue) { error!("failed to process ctrl queue: {:?}", e); return true; } @@ -125,7 +124,7 @@ struct NetEpollHandler { interrupt_cb: Arc, kill_evt: EventFd, pause_evt: EventFd, - queue_pair: Vec, + queue_pair: Vec>>, queue_evt_pair: Vec, // Always generate interrupts until the driver has signalled to the device. // This mitigates a problem with interrupts from tap events being "lost" upon @@ -135,7 +134,10 @@ struct NetEpollHandler { } impl NetEpollHandler { - fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> { + fn signal_used_queue( + &self, + queue: &Queue>, + ) -> result::Result<(), DeviceError> { self.interrupt_cb .trigger(&VirtioInterruptType::Queue, Some(queue)) .map_err(|e| { @@ -235,8 +237,11 @@ impl NetEpollHandler { // If there are some already available descriptors on the RX queue, // then we can start the thread while listening onto the TAP. if self.queue_pair[0] - .available_descriptors(&self.net.mem.as_ref().unwrap().memory()) - .unwrap() + .used_idx(Ordering::Acquire) + .map_err(EpollHelperError::QueueRingIndex)? + < self.queue_pair[0] + .avail_idx(Ordering::Acquire) + .map_err(EpollHelperError::QueueRingIndex)? { helper.add_event(self.net.tap.as_raw_fd(), RX_TAP_EVENT)?; self.net.rx_tap_listening = true; @@ -549,9 +554,9 @@ impl VirtioDevice for Net { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - mut queues: Vec, + mut queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -563,7 +568,6 @@ impl VirtioDevice for Net { let (kill_evt, pause_evt) = self.common.dup_eventfds(); let mut ctrl_handler = NetCtrlEpollHandler { - mem: mem.clone(), kill_evt, pause_evt, ctrl_q: CtrlQueue::new(self.taps.clone()), @@ -632,7 +636,6 @@ impl VirtioDevice for Net { let mut handler = NetEpollHandler { net: NetQueuePair { - mem: Some(mem.clone()), tap_for_write_epoll: tap.clone(), tap, rx, diff --git a/virtio-devices/src/pmem.rs b/virtio-devices/src/pmem.rs index 75b65e9ad..447b79ef8 100644 --- a/virtio-devices/src/pmem.rs +++ b/virtio-devices/src/pmem.rs @@ -8,9 +8,9 @@ use super::Error as DeviceError; use super::{ - ActivateError, ActivateResult, DescriptorChain, EpollHelper, EpollHelperError, - EpollHelperHandler, Queue, UserspaceMapping, VirtioCommon, VirtioDevice, VirtioDeviceType, - EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1, + ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, + UserspaceMapping, VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, + VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1, }; use crate::seccomp_filters::Thread; use crate::thread_helper::spawn_virtio_thread; @@ -27,10 +27,8 @@ use std::sync::atomic::AtomicBool; use std::sync::{Arc, Barrier}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; -use vm_memory::{ - Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, - GuestMemoryError, -}; +use virtio_queue::{DescriptorChain, Queue}; +use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryAtomic, GuestMemoryError}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; use vmm_sys_util::eventfd::EventFd; @@ -116,48 +114,48 @@ struct Request { impl Request { fn parse( - avail_desc: &DescriptorChain, - mem: &GuestMemoryMmap, + desc_chain: &mut DescriptorChain>, ) -> result::Result { - // The head contains the request type which MUST be readable. - if avail_desc.is_write_only() { + let desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?; + // The descriptor contains the request type which MUST be readable. + if desc.is_write_only() { return Err(Error::UnexpectedWriteOnlyDescriptor); } - if avail_desc.len as usize != size_of::() { + if desc.len() as usize != size_of::() { return Err(Error::InvalidRequest); } - let request: VirtioPmemReq = mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?; + let request: VirtioPmemReq = desc_chain + .memory() + .read_obj(desc.addr()) + .map_err(Error::GuestMemory)?; let request_type = match request.type_ { VIRTIO_PMEM_REQ_TYPE_FLUSH => RequestType::Flush, _ => return Err(Error::InvalidRequest), }; - let status_desc = avail_desc - .next_descriptor() - .ok_or(Error::DescriptorChainTooShort)?; + let status_desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?; // The status MUST always be writable if !status_desc.is_write_only() { return Err(Error::UnexpectedReadOnlyDescriptor); } - if (status_desc.len as usize) < size_of::() { + if (status_desc.len() as usize) < size_of::() { return Err(Error::BufferLengthTooSmall); } Ok(Request { type_: request_type, - status_addr: status_desc.addr, + status_addr: status_desc.addr(), }) } } struct PmemEpollHandler { - queue: Queue, - mem: GuestMemoryAtomic, + queue: Queue>, disk: File, interrupt_cb: Arc, queue_evt: EventFd, @@ -169,9 +167,8 @@ impl PmemEpollHandler { fn process_queue(&mut self) -> bool { let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in self.queue.iter(&mem) { - let len = match Request::parse(&avail_desc, &mem) { + for mut desc_chain in self.queue.iter().unwrap() { + let len = match Request::parse(&mut desc_chain) { Ok(ref req) if (req.type_ == RequestType::Flush) => { let status_code = match self.disk.sync_all() { Ok(()) => VIRTIO_PMEM_RESP_TYPE_OK, @@ -182,7 +179,7 @@ impl PmemEpollHandler { }; let resp = VirtioPmemResp { ret: status_code }; - match mem.write_obj(resp, req.status_addr) { + match desc_chain.memory().write_obj(resp, req.status_addr) { Ok(_) => size_of::() as u32, Err(e) => { error!("bad guest memory address: {}", e); @@ -201,12 +198,12 @@ impl PmemEpollHandler { } }; - used_desc_heads[used_count] = (avail_desc.index, len); + used_desc_heads[used_count] = (desc_chain.head_index(), len); used_count += 1; } for &(desc_index, len) in &used_desc_heads[..used_count] { - self.queue.add_used(&mem, desc_index, len); + self.queue.add_used(desc_index, len).unwrap(); } used_count > 0 } @@ -369,9 +366,9 @@ impl VirtioDevice for Pmem { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - mut queues: Vec, + mut queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -383,7 +380,6 @@ impl VirtioDevice for Pmem { })?; let mut handler = PmemEpollHandler { queue: queues.remove(0), - mem, disk, interrupt_cb, queue_evt: queue_evts.remove(0), diff --git a/virtio-devices/src/rng.rs b/virtio-devices/src/rng.rs index 0c56d1b0c..7d441ca52 100644 --- a/virtio-devices/src/rng.rs +++ b/virtio-devices/src/rng.rs @@ -4,8 +4,8 @@ use super::Error as DeviceError; use super::{ - ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, - VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IOMMU_PLATFORM, + ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, + VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1, }; use crate::seccomp_filters::Thread; @@ -21,7 +21,8 @@ use std::sync::atomic::AtomicBool; use std::sync::{Arc, Barrier}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; -use vm_memory::{Bytes, GuestAddressSpace, GuestMemoryAtomic}; +use virtio_queue::Queue; +use vm_memory::{Bytes, GuestMemoryAtomic}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; use vmm_sys_util::eventfd::EventFd; @@ -33,8 +34,7 @@ const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE]; const QUEUE_AVAIL_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; struct RngEpollHandler { - queues: Vec, - mem: GuestMemoryAtomic, + queues: Vec>>, random_file: File, interrupt_cb: Arc, queue_evt: EventFd, @@ -48,31 +48,28 @@ impl RngEpollHandler { let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in queue.iter(&mem) { + for mut desc_chain in queue.iter().unwrap() { + let desc = desc_chain.next().unwrap(); let mut len = 0; // Drivers can only read from the random device. - if avail_desc.is_write_only() { + if desc.is_write_only() { // Fill the read with data from the random device on the host. - if mem - .read_from( - avail_desc.addr, - &mut self.random_file, - avail_desc.len as usize, - ) + if desc_chain + .memory() + .read_from(desc.addr(), &mut self.random_file, desc.len() as usize) .is_ok() { - len = avail_desc.len; + len = desc.len(); } } - used_desc_heads[used_count] = (avail_desc.index, len); + used_desc_heads[used_count] = (desc_chain.head_index(), len); used_count += 1; } for &(desc_index, len) in &used_desc_heads[..used_count] { - queue.add_used(&mem, desc_index, len); + queue.add_used(desc_index, len).unwrap(); } used_count > 0 } @@ -213,9 +210,9 @@ impl VirtioDevice for Rng { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -228,7 +225,6 @@ impl VirtioDevice for Rng { })?; let mut handler = RngEpollHandler { queues, - mem, random_file, interrupt_cb, queue_evt: queue_evts.remove(0), diff --git a/virtio-devices/src/transport/pci_common_config.rs b/virtio-devices/src/transport/pci_common_config.rs index f769ad6de..9d9f2545e 100644 --- a/virtio-devices/src/transport/pci_common_config.rs +++ b/virtio-devices/src/transport/pci_common_config.rs @@ -6,13 +6,14 @@ // // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause -use crate::{Queue, VirtioDevice}; +use crate::{GuestMemoryMmap, VirtioDevice}; use byteorder::{ByteOrder, LittleEndian}; use std::sync::atomic::{AtomicU16, Ordering}; use std::sync::{Arc, Mutex}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; -use vm_memory::GuestAddress; +use virtio_queue::Queue; +use vm_memory::{GuestAddress, GuestMemoryAtomic}; use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable, VersionMapped}; #[derive(Clone, Versionize)] @@ -83,7 +84,7 @@ impl VirtioPciCommonConfig { &mut self, offset: u64, data: &mut [u8], - queues: &mut Vec, + queues: &mut Vec>>, device: Arc>, ) { assert!(data.len() <= 8); @@ -113,7 +114,7 @@ impl VirtioPciCommonConfig { &mut self, offset: u64, data: &[u8], - queues: &mut Vec, + queues: &mut Vec>>, device: Arc>, ) { assert!(data.len() <= 8); @@ -152,16 +153,20 @@ impl VirtioPciCommonConfig { } } - fn read_common_config_word(&self, offset: u64, queues: &[Queue]) -> u16 { + fn read_common_config_word( + &self, + offset: u64, + queues: &[Queue>], + ) -> u16 { debug!("read_common_config_word: offset 0x{:x}", offset); match offset { 0x10 => self.msix_config.load(Ordering::Acquire), 0x12 => queues.len() as u16, // num_queues 0x16 => self.queue_select, - 0x18 => self.with_queue(queues, |q| q.size).unwrap_or(0), - 0x1a => self.with_queue(queues, |q| q.vector).unwrap_or(0), + 0x18 => self.with_queue(queues, |q| q.state.size).unwrap_or(0), + 0x1a => self.with_queue(queues, |q| q.state.vector).unwrap_or(0), 0x1c => { - if self.with_queue(queues, |q| q.ready).unwrap_or(false) { + if self.with_queue(queues, |q| q.state.ready).unwrap_or(false) { 1 } else { 0 @@ -175,13 +180,18 @@ impl VirtioPciCommonConfig { } } - fn write_common_config_word(&mut self, offset: u64, value: u16, queues: &mut Vec) { + fn write_common_config_word( + &mut self, + offset: u64, + value: u16, + queues: &mut Vec>>, + ) { debug!("write_common_config_word: offset 0x{:x}", offset); match offset { 0x10 => self.msix_config.store(value, Ordering::Release), 0x16 => self.queue_select = value, - 0x18 => self.with_queue_mut(queues, |q| q.size = value), - 0x1a => self.with_queue_mut(queues, |q| q.vector = value), + 0x18 => self.with_queue_mut(queues, |q| q.state.size = value), + 0x1a => self.with_queue_mut(queues, |q| q.state.vector = value), 0x1c => self.with_queue_mut(queues, |q| q.enable(value == 1)), _ => { warn!("invalid virtio register word write: 0x{:x}", offset); @@ -215,7 +225,7 @@ impl VirtioPciCommonConfig { &mut self, offset: u64, value: u32, - queues: &mut Vec, + queues: &mut Vec>>, device: Arc>, ) { debug!("write_common_config_dword: offset 0x{:x}", offset); @@ -242,12 +252,12 @@ impl VirtioPciCommonConfig { ); } } - 0x20 => self.with_queue_mut(queues, |q| lo(&mut q.desc_table, value)), - 0x24 => self.with_queue_mut(queues, |q| hi(&mut q.desc_table, value)), - 0x28 => self.with_queue_mut(queues, |q| lo(&mut q.avail_ring, value)), - 0x2c => self.with_queue_mut(queues, |q| hi(&mut q.avail_ring, value)), - 0x30 => self.with_queue_mut(queues, |q| lo(&mut q.used_ring, value)), - 0x34 => self.with_queue_mut(queues, |q| hi(&mut q.used_ring, value)), + 0x20 => self.with_queue_mut(queues, |q| lo(&mut q.state.desc_table, value)), + 0x24 => self.with_queue_mut(queues, |q| hi(&mut q.state.desc_table, value)), + 0x28 => self.with_queue_mut(queues, |q| lo(&mut q.state.avail_ring, value)), + 0x2c => self.with_queue_mut(queues, |q| hi(&mut q.state.avail_ring, value)), + 0x30 => self.with_queue_mut(queues, |q| lo(&mut q.state.used_ring, value)), + 0x34 => self.with_queue_mut(queues, |q| hi(&mut q.state.used_ring, value)), _ => { warn!("invalid virtio register dword write: 0x{:x}", offset); } @@ -259,26 +269,39 @@ impl VirtioPciCommonConfig { 0 // Assume the guest has no reason to read write-only registers. } - fn write_common_config_qword(&mut self, offset: u64, value: u64, queues: &mut Vec) { + fn write_common_config_qword( + &mut self, + offset: u64, + value: u64, + queues: &mut Vec>>, + ) { debug!("write_common_config_qword: offset 0x{:x}", offset); match offset { - 0x20 => self.with_queue_mut(queues, |q| q.desc_table = GuestAddress(value)), - 0x28 => self.with_queue_mut(queues, |q| q.avail_ring = GuestAddress(value)), - 0x30 => self.with_queue_mut(queues, |q| q.used_ring = GuestAddress(value)), + 0x20 => self.with_queue_mut(queues, |q| q.state.desc_table = GuestAddress(value)), + 0x28 => self.with_queue_mut(queues, |q| q.state.avail_ring = GuestAddress(value)), + 0x30 => self.with_queue_mut(queues, |q| q.state.used_ring = GuestAddress(value)), _ => { warn!("invalid virtio register qword write: 0x{:x}", offset); } } } - fn with_queue(&self, queues: &[Queue], f: F) -> Option + fn with_queue( + &self, + queues: &[Queue>], + f: F, + ) -> Option where - F: FnOnce(&Queue) -> U, + F: FnOnce(&Queue>) -> U, { queues.get(self.queue_select as usize).map(f) } - fn with_queue_mut(&self, queues: &mut Vec, f: F) { + fn with_queue_mut>)>( + &self, + queues: &mut Vec>>, + f: F, + ) { if let Some(queue) = queues.get_mut(self.queue_select as usize) { f(queue); } @@ -308,6 +331,7 @@ mod tests { use crate::GuestMemoryMmap; use crate::{ActivateResult, VirtioInterrupt}; use std::sync::Arc; + use virtio_queue::Queue; use vm_memory::GuestMemoryAtomic; use vmm_sys_util::eventfd::EventFd; @@ -326,7 +350,7 @@ mod tests { &mut self, _mem: GuestMemoryAtomic, _interrupt_evt: Arc, - _queues: Vec, + _queues: Vec>>, _queue_evts: Vec, ) -> ActivateResult { Ok(()) diff --git a/virtio-devices/src/transport/pci_device.rs b/virtio-devices/src/transport/pci_device.rs index 5957e4d90..ed6d6a6fc 100644 --- a/virtio-devices/src/transport/pci_device.rs +++ b/virtio-devices/src/transport/pci_device.rs @@ -10,7 +10,7 @@ use super::VirtioPciCommonConfig; use crate::transport::VirtioTransport; use crate::GuestMemoryMmap; use crate::{ - ActivateResult, Queue, VirtioDevice, VirtioDeviceType, VirtioInterrupt, VirtioInterruptType, + ActivateResult, VirtioDevice, VirtioDeviceType, VirtioInterrupt, VirtioInterruptType, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER, DEVICE_DRIVER_OK, DEVICE_FAILED, DEVICE_FEATURES_OK, DEVICE_INIT, }; @@ -24,30 +24,28 @@ use pci::{ use std::any::Any; use std::cmp; use std::io::Write; -use std::num::Wrapping; use std::result; use std::sync::atomic::{AtomicBool, AtomicU16, AtomicUsize, Ordering}; use std::sync::{Arc, Barrier, Mutex}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; +use virtio_queue::AccessPlatform; +use virtio_queue::{defs::VIRTQ_MSI_NO_VECTOR, Error as QueueError, Queue}; use vm_allocator::SystemAllocator; use vm_device::interrupt::{ InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig, }; use vm_device::BusDevice; -use vm_memory::{ - Address, ByteValued, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestUsize, Le32, -}; +use vm_memory::{Address, ByteValued, GuestAddress, GuestMemoryAtomic, GuestUsize, Le32}; use vm_migration::{ Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable, VersionMapped, }; -use vm_virtio::{queue, VirtioIommuRemapping, VIRTIO_MSI_NO_VECTOR}; use vmm_sys_util::{errno::Result, eventfd::EventFd}; #[derive(Debug)] enum Error { /// Failed to retrieve queue ring's index. - QueueRingIndex(queue::Error), + QueueRingIndex(QueueError), } #[allow(clippy::enum_variant_names)] @@ -309,7 +307,7 @@ pub struct VirtioPciDevice { interrupt_source_group: Arc, // virtio queues - queues: Vec, + queues: Vec>>, queue_evts: Vec, // Guest memory @@ -348,7 +346,7 @@ impl VirtioPciDevice { memory: GuestMemoryAtomic, device: Arc>, msix_num: u16, - iommu_mapping_cb: Option>, + access_platform: Option>, interrupt_manager: &Arc>, pci_device_bdf: u32, activate_evt: EventFd, @@ -363,8 +361,11 @@ impl VirtioPciDevice { .queue_max_sizes() .iter() .map(|&s| { - let mut queue = Queue::new(s); - queue.iommu_mapping_cb = iommu_mapping_cb.clone(); + let mut queue = Queue::< + GuestMemoryAtomic, + virtio_queue::QueueState>, + >::new(memory.clone(), s); + queue.state.access_platform = access_platform.clone(); queue }) .collect(); @@ -432,7 +433,7 @@ impl VirtioPciDevice { device_feature_select: 0, driver_feature_select: 0, queue_select: 0, - msix_config: Arc::new(AtomicU16::new(VIRTIO_MSI_NO_VECTOR)), + msix_config: Arc::new(AtomicU16::new(VIRTQ_MSI_NO_VECTOR)), }, msix_config, msix_num, @@ -472,13 +473,13 @@ impl VirtioPciDevice { .queues .iter() .map(|q| QueueState { - max_size: q.max_size, - size: q.size, - ready: q.ready, - vector: q.vector, - desc_table: q.desc_table.0, - avail_ring: q.avail_ring.0, - used_ring: q.used_ring.0, + max_size: q.max_size(), + size: q.state.size, + ready: q.state.ready, + vector: q.state.vector, + desc_table: q.state.desc_table.0, + avail_ring: q.state.avail_ring.0, + used_ring: q.state.used_ring.0, }) .collect(), } @@ -491,27 +492,25 @@ impl VirtioPciDevice { .store(state.interrupt_status, Ordering::Release); // Update virtqueues indexes for both available and used rings. - if let Some(mem) = self.memory.as_ref() { - let mem = mem.memory(); - for (i, queue) in self.queues.iter_mut().enumerate() { - queue.max_size = state.queues[i].max_size; - queue.size = state.queues[i].size; - queue.ready = state.queues[i].ready; - queue.vector = state.queues[i].vector; - queue.desc_table = GuestAddress(state.queues[i].desc_table); - queue.avail_ring = GuestAddress(state.queues[i].avail_ring); - queue.used_ring = GuestAddress(state.queues[i].used_ring); - queue.next_avail = Wrapping( - queue - .used_index_from_memory(&mem) - .map_err(Error::QueueRingIndex)?, - ); - queue.next_used = Wrapping( - queue - .used_index_from_memory(&mem) - .map_err(Error::QueueRingIndex)?, - ); - } + for (i, queue) in self.queues.iter_mut().enumerate() { + queue.state.size = state.queues[i].size; + queue.state.ready = state.queues[i].ready; + queue.state.vector = state.queues[i].vector; + queue.state.desc_table = GuestAddress(state.queues[i].desc_table); + queue.state.avail_ring = GuestAddress(state.queues[i].avail_ring); + queue.state.used_ring = GuestAddress(state.queues[i].used_ring); + queue.set_next_avail( + queue + .used_idx(Ordering::Acquire) + .map_err(Error::QueueRingIndex)? + .0, + ); + queue.set_next_used( + queue + .used_idx(Ordering::Acquire) + .map_err(Error::QueueRingIndex)? + .0, + ); } Ok(()) @@ -673,10 +672,10 @@ impl VirtioPciDevice { let mut device = self.device.lock().unwrap(); let mut queue_evts = Vec::new(); let mut queues = self.queues.clone(); - queues.retain(|q| q.ready); + queues.retain(|q| q.state.ready); for (i, queue) in queues.iter().enumerate() { queue_evts.push(self.queue_evts[i].try_clone().unwrap()); - if !queue.is_valid(&mem.memory()) { + if !queue.is_valid() { error!("Queue {} is not valid", i); } } @@ -743,20 +742,20 @@ impl VirtioInterrupt for VirtioInterruptMsix { fn trigger( &self, int_type: &VirtioInterruptType, - queue: Option<&Queue>, + queue: Option<&Queue>>, ) -> std::result::Result<(), std::io::Error> { let vector = match int_type { VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire), VirtioInterruptType::Queue => { if let Some(q) = queue { - q.vector + q.state.vector } else { 0 } } }; - if vector == VIRTIO_MSI_NO_VECTOR { + if vector == VIRTQ_MSI_NO_VECTOR { return Ok(()); } @@ -776,12 +775,16 @@ impl VirtioInterrupt for VirtioInterruptMsix { .trigger(vector as InterruptIndex) } - fn notifier(&self, int_type: &VirtioInterruptType, queue: Option<&Queue>) -> Option { + fn notifier( + &self, + int_type: &VirtioInterruptType, + queue: Option<&Queue>>, + ) -> Option { let vector = match int_type { VirtioInterruptType::Config => self.config_vector.load(Ordering::Acquire), VirtioInterruptType::Queue => { if let Some(q) = queue { - q.vector + q.state.vector } else { 0 } diff --git a/virtio-devices/src/vhost_user/blk.rs b/virtio-devices/src/vhost_user/blk.rs index 0cfdd6889..a6f0533ab 100644 --- a/virtio-devices/src/vhost_user/blk.rs +++ b/virtio-devices/src/vhost_user/blk.rs @@ -1,7 +1,7 @@ // Copyright 2019 Intel Corporation. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use super::super::{ActivateResult, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType}; +use super::super::{ActivateResult, VirtioCommon, VirtioDevice, VirtioDeviceType}; use super::vu_common_ctrl::{VhostUserConfig, VhostUserHandle}; use super::{Error, Result, DEFAULT_VIRTIO_FEATURES}; use crate::seccomp_filters::Thread; @@ -28,6 +28,7 @@ use virtio_bindings::bindings::virtio_blk::{ VIRTIO_BLK_F_GEOMETRY, VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_RO, VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_WRITE_ZEROES, }; +use virtio_queue::Queue; use vm_memory::{ByteValued, GuestMemoryAtomic}; use vm_migration::{ protocol::MemoryRangeTable, Migratable, MigratableError, Pausable, Snapshot, Snapshottable, @@ -279,7 +280,7 @@ impl VirtioDevice for Blk { &mut self, mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; diff --git a/virtio-devices/src/vhost_user/fs.rs b/virtio-devices/src/vhost_user/fs.rs index 2834e152c..83d08b105 100644 --- a/virtio-devices/src/vhost_user/fs.rs +++ b/virtio-devices/src/vhost_user/fs.rs @@ -7,8 +7,8 @@ use crate::seccomp_filters::Thread; use crate::thread_helper::spawn_virtio_thread; use crate::vhost_user::VhostUserCommon; use crate::{ - ActivateError, ActivateResult, Queue, UserspaceMapping, VirtioCommon, VirtioDevice, - VirtioDeviceType, VirtioInterrupt, VirtioSharedMemoryList, + ActivateError, ActivateResult, UserspaceMapping, VirtioCommon, VirtioDevice, VirtioDeviceType, + VirtioInterrupt, VirtioSharedMemoryList, }; use crate::{GuestMemoryMmap, GuestRegionMmap, MmapRegion}; use libc::{self, c_void, off64_t, pread64, pwrite64}; @@ -27,6 +27,7 @@ use vhost::vhost_user::message::{ use vhost::vhost_user::{ HandlerResult, MasterReqHandler, VhostUserMaster, VhostUserMasterReqHandler, }; +use virtio_queue::Queue; use vm_memory::{ Address, ByteValued, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic, }; @@ -501,7 +502,7 @@ impl VirtioDevice for Fs { &mut self, mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; diff --git a/virtio-devices/src/vhost_user/mod.rs b/virtio-devices/src/vhost_user/mod.rs index 30824ba59..1e2509fab 100644 --- a/virtio-devices/src/vhost_user/mod.rs +++ b/virtio-devices/src/vhost_user/mod.rs @@ -3,7 +3,7 @@ use crate::{ ActivateError, EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, - GuestRegionMmap, Queue, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, + GuestRegionMmap, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, VIRTIO_F_NOTIFICATION_DATA, VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, VIRTIO_F_RING_INDIRECT_DESC, VIRTIO_F_VERSION_1, }; @@ -18,12 +18,13 @@ use vhost::vhost_user::message::{ }; use vhost::vhost_user::{MasterReqHandler, VhostUserMasterReqHandler}; use vhost::Error as VhostError; +use virtio_queue::Error as QueueError; +use virtio_queue::Queue; use vm_memory::{ mmap::MmapRegionError, Address, Error as MmapError, GuestAddressSpace, GuestMemory, GuestMemoryAtomic, }; use vm_migration::{protocol::MemoryRangeTable, MigratableError, Snapshot, VersionMapped}; -use vm_virtio::Error as VirtioError; use vmm_sys_util::eventfd::EventFd; use vu_common_ctrl::VhostUserHandle; @@ -128,7 +129,7 @@ pub enum Error { /// Missing IrqFd MissingIrqFd, /// Failed getting the available index. - GetAvailableIndex(VirtioError), + GetAvailableIndex(QueueError), /// Migration is not supported by this vhost-user device. MigrationNotSupported, /// Failed creating memfd. @@ -166,7 +167,7 @@ pub struct VhostUserEpollHandler { pub mem: GuestMemoryAtomic, pub kill_evt: EventFd, pub pause_evt: EventFd, - pub queues: Vec, + pub queues: Vec>>, pub queue_evts: Vec, pub virtio_interrupt: Arc, pub acked_features: u64, @@ -298,7 +299,7 @@ impl VhostUserCommon { pub fn activate( &mut self, mem: GuestMemoryAtomic, - queues: Vec, + queues: Vec>>, queue_evts: Vec, interrupt_cb: Arc, acked_features: u64, diff --git a/virtio-devices/src/vhost_user/net.rs b/virtio-devices/src/vhost_user/net.rs index 787fc15e2..451f88674 100644 --- a/virtio-devices/src/vhost_user/net.rs +++ b/virtio-devices/src/vhost_user/net.rs @@ -6,9 +6,9 @@ use crate::thread_helper::spawn_virtio_thread; use crate::vhost_user::vu_common_ctrl::{VhostUserConfig, VhostUserHandle}; use crate::vhost_user::{Error, Result, VhostUserCommon}; use crate::{ - ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, VirtioCommon, - VirtioDevice, VirtioDeviceType, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, - VIRTIO_F_RING_EVENT_IDX, VIRTIO_F_VERSION_1, + ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, VirtioDevice, + VirtioDeviceType, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_RING_EVENT_IDX, + VIRTIO_F_VERSION_1, }; use crate::{GuestMemoryMmap, GuestRegionMmap}; use net_util::{build_net_config_space, CtrlQueue, MacAddr, VirtioNetConfig}; @@ -29,7 +29,8 @@ use virtio_bindings::bindings::virtio_net::{ VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_TSO6, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_MAC, VIRTIO_NET_F_MRG_RXBUF, }; -use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic}; +use virtio_queue::Queue; +use vm_memory::{ByteValued, GuestMemoryAtomic}; use vm_migration::{ protocol::MemoryRangeTable, Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable, VersionMapped, @@ -62,7 +63,7 @@ pub struct NetCtrlEpollHandler { pub pause_evt: EventFd, pub ctrl_q: CtrlQueue, pub queue_evt: EventFd, - pub queue: Queue, + pub queue: Queue>, } impl NetCtrlEpollHandler { @@ -84,12 +85,11 @@ impl EpollHelperHandler for NetCtrlEpollHandler { let ev_type = event.data as u16; match ev_type { CTRL_QUEUE_EVENT => { - let mem = self.mem.memory(); if let Err(e) = self.queue_evt.read() { error!("failed to get ctl queue event: {:?}", e); return true; } - if let Err(e) = self.ctrl_q.process(&mem, &mut self.queue) { + if let Err(e) = self.ctrl_q.process(&mut self.queue) { error!("failed to process ctrl queue: {:?}", e); return true; } @@ -308,7 +308,7 @@ impl VirtioDevice for Net { &mut self, mem: GuestMemoryAtomic, interrupt_cb: Arc, - mut queues: Vec, + mut queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; diff --git a/virtio-devices/src/vhost_user/vu_common_ctrl.rs b/virtio-devices/src/vhost_user/vu_common_ctrl.rs index 8b5d877bd..133cd8cbc 100644 --- a/virtio-devices/src/vhost_user/vu_common_ctrl.rs +++ b/virtio-devices/src/vhost_user/vu_common_ctrl.rs @@ -1,7 +1,6 @@ // Copyright 2019 Intel Corporation. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use super::super::{Descriptor, Queue}; use super::{Error, Result}; use crate::vhost_user::Inflight; use crate::{ @@ -13,6 +12,7 @@ use std::ffi; use std::fs::File; use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::os::unix::net::UnixListener; +use std::sync::atomic::Ordering; use std::sync::Arc; use std::thread::sleep; use std::time::{Duration, Instant}; @@ -23,7 +23,10 @@ use vhost::vhost_user::message::{ }; use vhost::vhost_user::{Master, MasterReqHandler, VhostUserMaster, VhostUserMasterReqHandler}; use vhost::{VhostBackend, VhostUserDirtyLogRegion, VhostUserMemoryRegionInfo, VringConfigData}; -use vm_memory::{Address, Error as MmapError, FileOffset, GuestMemory, GuestMemoryRegion}; +use virtio_queue::{Descriptor, Queue}; +use vm_memory::{ + Address, Error as MmapError, FileOffset, GuestMemory, GuestMemoryAtomic, GuestMemoryRegion, +}; use vm_migration::protocol::MemoryRangeTable; use vmm_sys_util::eventfd::EventFd; @@ -148,7 +151,7 @@ impl VhostUserHandle { pub fn setup_vhost_user( &mut self, mem: &GuestMemoryMmap, - queues: Vec, + queues: Vec>>, queue_evts: Vec, virtio_interrupt: &Arc, acked_features: u64, @@ -203,29 +206,37 @@ impl VhostUserHandle { let actual_size: usize = queue.actual_size().try_into().unwrap(); let config_data = VringConfigData { - queue_max_size: queue.get_max_size(), + queue_max_size: queue.max_size(), queue_size: queue.actual_size(), flags: 0u32, desc_table_addr: get_host_address_range( mem, - queue.desc_table, + queue.state.desc_table, actual_size * std::mem::size_of::(), ) .ok_or(Error::DescriptorTableAddress)? as u64, // The used ring is {flags: u16; idx: u16; virtq_used_elem [{id: u16, len: u16}; actual_size]}, // i.e. 4 + (4 + 4) * actual_size. - used_ring_addr: get_host_address_range(mem, queue.used_ring, 4 + actual_size * 8) - .ok_or(Error::UsedAddress)? as u64, + used_ring_addr: get_host_address_range( + mem, + queue.state.used_ring, + 4 + actual_size * 8, + ) + .ok_or(Error::UsedAddress)? as u64, // The used ring is {flags: u16; idx: u16; elem [u16; actual_size]}, // i.e. 4 + (2) * actual_size. - avail_ring_addr: get_host_address_range(mem, queue.avail_ring, 4 + actual_size * 2) - .ok_or(Error::AvailAddress)? as u64, + avail_ring_addr: get_host_address_range( + mem, + queue.state.avail_ring, + 4 + actual_size * 2, + ) + .ok_or(Error::AvailAddress)? as u64, log_addr: None, }; vrings_info.push(VringInfo { config_data, - used_guest_addr: queue.used_ring.raw_value(), + used_guest_addr: queue.state.used_ring.raw_value(), }); self.vu @@ -235,8 +246,9 @@ impl VhostUserHandle { .set_vring_base( queue_index, queue - .used_index_from_memory(mem) - .map_err(Error::GetAvailableIndex)?, + .avail_idx(Ordering::Acquire) + .map_err(Error::GetAvailableIndex)? + .0, ) .map_err(Error::VhostUserSetVringBase)?; @@ -317,7 +329,7 @@ impl VhostUserHandle { pub fn reinitialize_vhost_user( &mut self, mem: &GuestMemoryMmap, - queues: Vec, + queues: Vec>>, queue_evts: Vec, virtio_interrupt: &Arc, acked_features: u64, diff --git a/virtio-devices/src/vsock/csm/connection.rs b/virtio-devices/src/vsock/csm/connection.rs index 4ad96d8d9..9b8741ba8 100644 --- a/virtio-devices/src/vsock/csm/connection.rs +++ b/virtio-devices/src/vsock/csm/connection.rs @@ -818,8 +818,9 @@ mod tests { let mut handler_ctx = vsock_test_ctx.create_epoll_handler_context(); let stream = TestStream::new(); let mut pkt = VsockPacket::from_rx_virtq_head( - &handler_ctx.handler.queues[0] - .iter(&vsock_test_ctx.mem) + &mut handler_ctx.handler.queues[0] + .iter() + .unwrap() .next() .unwrap(), ) diff --git a/virtio-devices/src/vsock/device.rs b/virtio-devices/src/vsock/device.rs index 55e522897..74400c4fa 100644 --- a/virtio-devices/src/vsock/device.rs +++ b/virtio-devices/src/vsock/device.rs @@ -34,7 +34,7 @@ use crate::GuestMemoryMmap; use crate::VirtioInterrupt; use crate::{ thread_helper::spawn_virtio_thread, ActivateResult, EpollHelper, EpollHelperError, - EpollHelperHandler, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, + EpollHelperHandler, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1, }; use byteorder::{ByteOrder, LittleEndian}; @@ -47,7 +47,8 @@ use std::sync::atomic::AtomicBool; use std::sync::{Arc, Barrier, RwLock}; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; -use vm_memory::{GuestAddressSpace, GuestMemoryAtomic}; +use virtio_queue::Queue; +use vm_memory::GuestMemoryAtomic; use vm_migration::{ Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable, VersionMapped, }; @@ -86,7 +87,7 @@ pub const BACKEND_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4; /// pub struct VsockEpollHandler { pub mem: GuestMemoryAtomic, - pub queues: Vec, + pub queues: Vec>>, pub queue_evts: Vec, pub kill_evt: EventFd, pub pause_evt: EventFd, @@ -101,7 +102,10 @@ where /// Signal the guest driver that we've used some virtio buffers that it had previously made /// available. /// - fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> { + fn signal_used_queue( + &self, + queue: &Queue>, + ) -> result::Result<(), DeviceError> { debug!("vsock: raising IRQ"); self.interrupt_cb @@ -120,16 +124,17 @@ where let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in self.queues[0].iter(&mem) { - let used_len = match VsockPacket::from_rx_virtq_head(&avail_desc) { + + let mut avail_iter = self.queues[0].iter().map_err(DeviceError::QueueIterator)?; + for mut desc_chain in &mut avail_iter { + let used_len = match VsockPacket::from_rx_virtq_head(&mut desc_chain) { Ok(mut pkt) => { if self.backend.write().unwrap().recv_pkt(&mut pkt).is_ok() { pkt.hdr().len() as u32 + pkt.len() } else { // We are using a consuming iterator over the virtio buffers, so, if we can't // fill in this buffer, we'll need to undo the last iterator step. - self.queues[0].go_to_previous_position(); + avail_iter.go_to_previous_position(); break; } } @@ -139,12 +144,14 @@ where } }; - used_desc_heads[used_count] = (avail_desc.index, used_len); + used_desc_heads[used_count] = (desc_chain.head_index(), used_len); used_count += 1; } for &(desc_index, len) in &used_desc_heads[..used_count] { - self.queues[0].add_used(&mem, desc_index, len); + self.queues[0] + .add_used(desc_index, len) + .map_err(DeviceError::QueueAddUsed)?; } if used_count > 0 { @@ -162,29 +169,32 @@ where let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in self.queues[1].iter(&mem) { - let pkt = match VsockPacket::from_tx_virtq_head(&avail_desc) { + + let mut avail_iter = self.queues[1].iter().map_err(DeviceError::QueueIterator)?; + for mut desc_chain in &mut avail_iter { + let pkt = match VsockPacket::from_tx_virtq_head(&mut desc_chain) { Ok(pkt) => pkt, Err(e) => { error!("vsock: error reading TX packet: {:?}", e); - used_desc_heads[used_count] = (avail_desc.index, 0); + used_desc_heads[used_count] = (desc_chain.head_index(), 0); used_count += 1; continue; } }; if self.backend.write().unwrap().send_pkt(&pkt).is_err() { - self.queues[1].go_to_previous_position(); + avail_iter.go_to_previous_position(); break; } - used_desc_heads[used_count] = (avail_desc.index, 0); + used_desc_heads[used_count] = (desc_chain.head_index(), 0); used_count += 1; } for &(desc_index, len) in &used_desc_heads[..used_count] { - self.queues[1].add_used(&mem, desc_index, len); + self.queues[1] + .add_used(desc_index, len) + .map_err(DeviceError::QueueAddUsed)?; } if used_count > 0 { @@ -417,7 +427,7 @@ where &mut self, mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -578,12 +588,18 @@ mod tests { other => panic!("{:?}", other), } + let memory = GuestMemoryAtomic::new(ctx.mem.clone()); + // Test a correct activation. ctx.device .activate( - GuestMemoryAtomic::new(ctx.mem.clone()), + memory.clone(), Arc::new(NoopVirtioInterrupt {}), - vec![Queue::new(256), Queue::new(256), Queue::new(256)], + vec![ + Queue::new(memory.clone(), 256), + Queue::new(memory.clone(), 256), + Queue::new(memory, 256), + ], vec![ EventFd::new(EFD_NONBLOCK).unwrap(), EventFd::new(EFD_NONBLOCK).unwrap(), @@ -599,8 +615,9 @@ mod tests { { let test_ctx = TestContext::new(); let ctx = test_ctx.create_epoll_handler_context(); + let memory = GuestMemoryAtomic::new(test_ctx.mem.clone()); - let queue = Queue::new(256); + let queue = Queue::new(memory, 256); assert!(ctx.handler.signal_used_queue(&queue).is_ok()); } } diff --git a/virtio-devices/src/vsock/mod.rs b/virtio-devices/src/vsock/mod.rs index 2291c8995..ff6ead210 100644 --- a/virtio-devices/src/vsock/mod.rs +++ b/virtio-devices/src/vsock/mod.rs @@ -75,6 +75,8 @@ pub enum VsockError { GuestMemoryBounds, /// The vsock header descriptor length is too small. HdrDescTooSmall(u32), + /// The vsock header descriptor is expected, but missing. + HdrDescMissing, /// The vsock header `len` field holds an invalid value. InvalidPktLen(u32), /// A data fetch was attempted when no data was available. @@ -168,10 +170,9 @@ mod tests { use std::os::unix::io::AsRawFd; use std::path::PathBuf; use std::sync::{Arc, RwLock}; + use virtio_queue::{defs::VIRTQ_DESC_F_NEXT, defs::VIRTQ_DESC_F_WRITE, Queue}; use vm_memory::{GuestAddress, GuestMemoryAtomic}; use vm_virtio::queue::testing::VirtQueue as GuestQ; - use vm_virtio::queue::Queue; - use vm_virtio::queue::{VIRTQ_DESC_F_NEXT, VIRTQ_DESC_F_WRITE}; use vmm_sys_util::eventfd::EventFd; pub struct NoopVirtioInterrupt {} @@ -180,7 +181,7 @@ mod tests { fn trigger( &self, _int_type: &VirtioInterruptType, - _queue: Option<&Queue>, + _queue: Option<&Queue>>, ) -> std::result::Result<(), std::io::Error> { Ok(()) } diff --git a/virtio-devices/src/vsock/packet.rs b/virtio-devices/src/vsock/packet.rs index c06e76050..17ec59ccd 100644 --- a/virtio-devices/src/vsock/packet.rs +++ b/virtio-devices/src/vsock/packet.rs @@ -19,7 +19,9 @@ use byteorder::{ByteOrder, LittleEndian}; use super::defs; use super::{Result, VsockError}; -use crate::{get_host_address_range, DescriptorChain}; +use crate::{get_host_address_range, GuestMemoryMmap}; +use virtio_queue::DescriptorChain; +use vm_memory::GuestMemoryAtomic; // The vsock packet header is defined by the C struct: // @@ -103,7 +105,11 @@ impl VsockPacket { /// descriptor can optionally end the chain. Bounds and pointer checks are performed when /// creating the wrapper. /// - pub fn from_tx_virtq_head(head: &DescriptorChain) -> Result { + pub fn from_tx_virtq_head( + desc_chain: &mut DescriptorChain>, + ) -> Result { + let head = desc_chain.next().ok_or(VsockError::HdrDescMissing)?; + // All buffers in the TX queue must be readable. // if head.is_write_only() { @@ -111,12 +117,12 @@ impl VsockPacket { } // The packet header should fit inside the head descriptor. - if head.len < VSOCK_PKT_HDR_SIZE as u32 { - return Err(VsockError::HdrDescTooSmall(head.len)); + if head.len() < VSOCK_PKT_HDR_SIZE as u32 { + return Err(VsockError::HdrDescTooSmall(head.len())); } let mut pkt = Self { - hdr: get_host_address_range(head.mem, head.addr, VSOCK_PKT_HDR_SIZE) + hdr: get_host_address_range(desc_chain.memory(), head.addr(), VSOCK_PKT_HDR_SIZE) .ok_or(VsockError::GuestMemory)? as *mut u8, buf: None, buf_size: 0, @@ -134,7 +140,7 @@ impl VsockPacket { } // If the packet header showed a non-zero length, there should be a data descriptor here. - let buf_desc = head.next_descriptor().ok_or(VsockError::BufDescMissing)?; + let buf_desc = desc_chain.next().ok_or(VsockError::BufDescMissing)?; // TX data should be read-only. if buf_desc.is_write_only() { @@ -143,13 +149,13 @@ impl VsockPacket { // The data buffer should be large enough to fit the size of the data, as described by // the header descriptor. - if buf_desc.len < pkt.len() { + if buf_desc.len() < pkt.len() { return Err(VsockError::BufDescTooSmall); } - pkt.buf_size = buf_desc.len as usize; + pkt.buf_size = buf_desc.len() as usize; pkt.buf = Some( - get_host_address_range(buf_desc.mem, buf_desc.addr, pkt.buf_size) + get_host_address_range(desc_chain.memory(), buf_desc.addr(), pkt.buf_size) .ok_or(VsockError::GuestMemory)? as *mut u8, ); @@ -161,7 +167,11 @@ impl VsockPacket { /// There must be two descriptors in the chain, both writable: a header descriptor and a data /// descriptor. Bounds and pointer checks are performed when creating the wrapper. /// - pub fn from_rx_virtq_head(head: &DescriptorChain) -> Result { + pub fn from_rx_virtq_head( + desc_chain: &mut DescriptorChain>, + ) -> Result { + let head = desc_chain.next().ok_or(VsockError::HdrDescMissing)?; + // All RX buffers must be writable. // if !head.is_write_only() { @@ -169,22 +179,22 @@ impl VsockPacket { } // The packet header should fit inside the head descriptor. - if head.len < VSOCK_PKT_HDR_SIZE as u32 { - return Err(VsockError::HdrDescTooSmall(head.len)); + if head.len() < VSOCK_PKT_HDR_SIZE as u32 { + return Err(VsockError::HdrDescTooSmall(head.len())); } // All RX descriptor chains should have a header and a data descriptor. if !head.has_next() { return Err(VsockError::BufDescMissing); } - let buf_desc = head.next_descriptor().ok_or(VsockError::BufDescMissing)?; - let buf_size = buf_desc.len as usize; + let buf_desc = desc_chain.next().ok_or(VsockError::BufDescMissing)?; + let buf_size = buf_desc.len() as usize; Ok(Self { - hdr: get_host_address_range(head.mem, head.addr, VSOCK_PKT_HDR_SIZE) + hdr: get_host_address_range(desc_chain.memory(), head.addr(), VSOCK_PKT_HDR_SIZE) .ok_or(VsockError::GuestMemory)? as *mut u8, buf: Some( - get_host_address_range(buf_desc.mem, buf_desc.addr, buf_size) + get_host_address_range(desc_chain.memory(), buf_desc.addr(), buf_size) .ok_or(VsockError::GuestMemory)? as *mut u8, ), buf_size, @@ -343,9 +353,9 @@ mod tests { use super::*; use crate::vsock::defs::MAX_PKT_BUF_SIZE; use crate::GuestMemoryMmap; + use virtio_queue::defs::VIRTQ_DESC_F_WRITE; use vm_memory::GuestAddress; use vm_virtio::queue::testing::VirtqDesc as GuestQDesc; - use vm_virtio::queue::VIRTQ_DESC_F_WRITE; macro_rules! create_context { ($test_ctx:ident, $handler_ctx:ident) => { @@ -365,8 +375,9 @@ mod tests { }; ($test_ctx:expr, $handler_ctx:expr, $err:pat, $ctor:ident, $vq:expr) => { match VsockPacket::$ctor( - &$handler_ctx.handler.queues[$vq] - .iter(&$test_ctx.mem) + &mut $handler_ctx.handler.queues[$vq] + .iter() + .unwrap() .next() .unwrap(), ) { @@ -394,8 +405,9 @@ mod tests { create_context!(test_ctx, handler_ctx); let pkt = VsockPacket::from_tx_virtq_head( - &handler_ctx.handler.queues[1] - .iter(&test_ctx.mem) + &mut handler_ctx.handler.queues[1] + .iter() + .unwrap() .next() .unwrap(), ) @@ -430,8 +442,9 @@ mod tests { create_context!(test_ctx, handler_ctx); set_pkt_len(0, &handler_ctx.guest_txvq.dtable[0], &test_ctx.mem); let mut pkt = VsockPacket::from_tx_virtq_head( - &handler_ctx.handler.queues[1] - .iter(&test_ctx.mem) + &mut handler_ctx.handler.queues[1] + .iter() + .unwrap() .next() .unwrap(), ) @@ -486,8 +499,9 @@ mod tests { { create_context!(test_ctx, handler_ctx); let pkt = VsockPacket::from_rx_virtq_head( - &handler_ctx.handler.queues[0] - .iter(&test_ctx.mem) + &mut handler_ctx.handler.queues[0] + .iter() + .unwrap() .next() .unwrap(), ) @@ -541,8 +555,9 @@ mod tests { create_context!(test_ctx, handler_ctx); let mut pkt = VsockPacket::from_rx_virtq_head( - &handler_ctx.handler.queues[0] - .iter(&test_ctx.mem) + &mut handler_ctx.handler.queues[0] + .iter() + .unwrap() .next() .unwrap(), ) @@ -630,8 +645,9 @@ mod tests { fn test_packet_buf() { create_context!(test_ctx, handler_ctx); let mut pkt = VsockPacket::from_rx_virtq_head( - &handler_ctx.handler.queues[0] - .iter(&test_ctx.mem) + &mut handler_ctx.handler.queues[0] + .iter() + .unwrap() .next() .unwrap(), ) diff --git a/virtio-devices/src/vsock/unix/muxer.rs b/virtio-devices/src/vsock/unix/muxer.rs index 0f482d8b7..88e50f9fa 100644 --- a/virtio-devices/src/vsock/unix/muxer.rs +++ b/virtio-devices/src/vsock/unix/muxer.rs @@ -840,8 +840,9 @@ mod tests { let vsock_test_ctx = VsockTestContext::new(); let mut handler_ctx = vsock_test_ctx.create_epoll_handler_context(); let pkt = VsockPacket::from_rx_virtq_head( - &handler_ctx.handler.queues[0] - .iter(&vsock_test_ctx.mem) + &mut handler_ctx.handler.queues[0] + .iter() + .unwrap() .next() .unwrap(), ) diff --git a/virtio-devices/src/watchdog.rs b/virtio-devices/src/watchdog.rs index 3abc18c50..b8e2a45a5 100644 --- a/virtio-devices/src/watchdog.rs +++ b/virtio-devices/src/watchdog.rs @@ -7,8 +7,8 @@ use super::Error as DeviceError; use super::{ - ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, - VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, + ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, + VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1, }; use crate::seccomp_filters::Thread; use crate::thread_helper::spawn_virtio_thread; @@ -25,7 +25,8 @@ use std::sync::{Arc, Barrier, Mutex}; use std::time::Instant; use versionize::{VersionMap, Versionize, VersionizeResult}; use versionize_derive::Versionize; -use vm_memory::{Bytes, GuestAddressSpace, GuestMemoryAtomic}; +use virtio_queue::Queue; +use vm_memory::{Bytes, GuestMemoryAtomic}; use vm_migration::VersionMapped; use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; use vmm_sys_util::eventfd::EventFd; @@ -46,8 +47,7 @@ const WATCHDOG_TIMER_INTERVAL: i64 = 15; const WATCHDOG_TIMEOUT: u64 = WATCHDOG_TIMER_INTERVAL as u64 + 5; struct WatchdogEpollHandler { - queues: Vec, - mem: GuestMemoryAtomic, + queues: Vec>>, interrupt_cb: Arc, queue_evt: EventFd, kill_evt: EventFd, @@ -64,12 +64,13 @@ impl WatchdogEpollHandler { let queue = &mut self.queues[0]; let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize]; let mut used_count = 0; - let mem = self.mem.memory(); - for avail_desc in queue.iter(&mem) { + for mut desc_chain in queue.iter().unwrap() { + let desc = desc_chain.next().unwrap(); + let mut len = 0; - if avail_desc.is_write_only() && mem.write_obj(1u8, avail_desc.addr).is_ok() { - len = avail_desc.len; + if desc.is_write_only() && desc_chain.memory().write_obj(1u8, desc.addr()).is_ok() { + len = desc.len(); // If this is the first "ping" then setup the timer if self.last_ping_time.lock().unwrap().is_none() { info!( @@ -83,12 +84,12 @@ impl WatchdogEpollHandler { self.last_ping_time.lock().unwrap().replace(Instant::now()); } - used_desc_heads[used_count] = (avail_desc.index, len); + used_desc_heads[used_count] = (desc_chain.head_index(), len); used_count += 1; } for &(desc_index, len) in &used_desc_heads[..used_count] { - queue.add_used(&mem, desc_index, len); + queue.add_used(desc_index, len).unwrap(); } used_count > 0 } @@ -288,9 +289,9 @@ impl VirtioDevice for Watchdog { fn activate( &mut self, - mem: GuestMemoryAtomic, + _mem: GuestMemoryAtomic, interrupt_cb: Arc, - queues: Vec, + queues: Vec>>, mut queue_evts: Vec, ) -> ActivateResult { self.common.activate(&queues, &queue_evts, &interrupt_cb)?; @@ -308,7 +309,6 @@ impl VirtioDevice for Watchdog { let mut handler = WatchdogEpollHandler { queues, - mem, interrupt_cb, queue_evt: queue_evts.remove(0), kill_evt, diff --git a/vm-virtio/Cargo.toml b/vm-virtio/Cargo.toml index d94b4a1d7..1dbea9ccc 100644 --- a/vm-virtio/Cargo.toml +++ b/vm-virtio/Cargo.toml @@ -10,4 +10,5 @@ default = [] [dependencies] log = "0.4.14" virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] } +virtio-queue = { path = "../virtio-queue" } vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] } diff --git a/vm-virtio/src/lib.rs b/vm-virtio/src/lib.rs index 09575d865..bb5ce2598 100644 --- a/vm-virtio/src/lib.rs +++ b/vm-virtio/src/lib.rs @@ -10,17 +10,11 @@ //! Implements virtio queues -#[macro_use] -extern crate log; - use std::fmt; pub mod queue; pub use queue::*; -pub type VirtioIommuRemapping = - Box std::result::Result + Send + Sync>; - pub const VIRTIO_MSI_NO_VECTOR: u16 = 0xffff; // Types taken from linux/virtio_ids.h diff --git a/vm-virtio/src/queue.rs b/vm-virtio/src/queue.rs index fd441c800..3ed71ab39 100644 --- a/vm-virtio/src/queue.rs +++ b/vm-virtio/src/queue.rs @@ -8,718 +8,12 @@ // // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause -use crate::{VirtioIommuRemapping, VIRTIO_MSI_NO_VECTOR}; -use std::cmp::min; -use std::convert::TryInto; -use std::fmt::{self, Display}; -use std::num::Wrapping; -use std::sync::atomic::{fence, Ordering}; -use std::sync::Arc; -use vm_memory::{ - bitmap::AtomicBitmap, Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError, - GuestUsize, -}; - -pub const VIRTQ_DESC_F_NEXT: u16 = 0x1; -pub const VIRTQ_DESC_F_WRITE: u16 = 0x2; -pub const VIRTQ_DESC_F_INDIRECT: u16 = 0x4; - -type GuestMemoryMmap = vm_memory::GuestMemoryMmap; - -#[derive(Debug)] -pub enum Error { - GuestMemoryError, - InvalidIndirectDescriptor, - InvalidChain, - InvalidOffset(u64), - InvalidRingIndexFromMemory(GuestMemoryError), -} - -impl Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::Error::*; - - match self { - GuestMemoryError => write!(f, "error accessing guest memory"), - InvalidChain => write!(f, "invalid descriptor chain"), - InvalidIndirectDescriptor => write!(f, "invalid indirect descriptor"), - InvalidOffset(o) => write!(f, "invalid offset {}", o), - InvalidRingIndexFromMemory(e) => write!(f, "invalid ring index from memory: {}", e), - } - } -} - -// GuestMemoryMmap::read_obj() will be used to fetch the descriptor, -// which has an explicit constraint that the entire descriptor doesn't -// cross the page boundary. Otherwise the descriptor may be splitted into -// two mmap regions which causes failure of GuestMemoryMmap::read_obj(). -// -// The Virtio Spec 1.0 defines the alignment of VirtIO descriptor is 16 bytes, -// which fulfills the explicit constraint of GuestMemoryMmap::read_obj(). - -/// An iterator over a single descriptor chain. Not to be confused with AvailIter, -/// which iterates over the descriptor chain heads in a queue. -pub struct DescIter<'a> { - next: Option>, -} - -impl<'a> DescIter<'a> { - /// Returns an iterator that only yields the readable descriptors in the chain. - pub fn readable(self) -> impl Iterator> { - self.filter(|d| !d.is_write_only()) - } - - /// Returns an iterator that only yields the writable descriptors in the chain. - pub fn writable(self) -> impl Iterator> { - self.filter(DescriptorChain::is_write_only) - } -} - -impl<'a> Iterator for DescIter<'a> { - type Item = DescriptorChain<'a>; - - fn next(&mut self) -> Option { - if let Some(current) = self.next.take() { - self.next = current.next_descriptor(); - Some(current) - } else { - None - } - } -} - -/// A virtio descriptor constraints with C representative. -#[repr(C)] -#[derive(Default, Clone, Copy)] -pub struct Descriptor { - addr: u64, - len: u32, - flags: u16, - next: u16, -} - -unsafe impl ByteValued for Descriptor {} - -/// A virtio descriptor head, not tied to a GuestMemoryMmap. -pub struct DescriptorHead { - desc_table: GuestAddress, - table_size: u16, - index: u16, - iommu_mapping_cb: Option>, -} - -/// A virtio descriptor chain. -#[derive(Clone)] -pub struct DescriptorChain<'a> { - desc_table: GuestAddress, - table_size: u16, - ttl: u16, // used to prevent infinite chain cycles - iommu_mapping_cb: Option>, - - /// Reference to guest memory - pub mem: &'a GuestMemoryMmap, - - /// Index into the descriptor table - pub index: u16, - - /// Guest physical address of device specific data - pub addr: GuestAddress, - - /// Length of device specific data - pub len: u32, - - /// Includes next, write, and indirect bits - pub flags: u16, - - /// Index into the descriptor table of the next descriptor if flags has - /// the next bit set - pub next: u16, -} - -impl<'a> DescriptorChain<'a> { - pub fn checked_new( - mem: &GuestMemoryMmap, - desc_table: GuestAddress, - table_size: u16, - index: u16, - iommu_mapping_cb: Option>, - ) -> Option { - if index >= table_size { - return None; - } - - let desc_head = match mem.checked_offset(desc_table, (index as usize) * 16) { - Some(a) => a, - None => return None, - }; - mem.checked_offset(desc_head, 16)?; - - // These reads can't fail unless Guest memory is hopelessly broken. - let desc = match mem.read_obj::(desc_head) { - Ok(ret) => ret, - Err(_) => { - // TODO log address - error!("Failed to read from memory"); - return None; - } - }; - - // Translate address if necessary - let desc_addr = if let Some(iommu_mapping_cb) = &iommu_mapping_cb { - (iommu_mapping_cb)(desc.addr).unwrap() - } else { - desc.addr - }; - - let chain = DescriptorChain { - mem, - desc_table, - table_size, - ttl: table_size, - index, - addr: GuestAddress(desc_addr), - len: desc.len, - flags: desc.flags, - next: desc.next, - iommu_mapping_cb, - }; - - if chain.is_valid() { - Some(chain) - } else { - None - } - } - - pub fn new_from_indirect(&self) -> Result { - if !self.is_indirect() { - return Err(Error::InvalidIndirectDescriptor); - } - - let desc_head = self.addr; - self.mem - .checked_offset(desc_head, 16) - .ok_or(Error::GuestMemoryError)?; - - // These reads can't fail unless Guest memory is hopelessly broken. - let desc = match self.mem.read_obj::(desc_head) { - Ok(ret) => ret, - Err(_) => return Err(Error::GuestMemoryError), - }; - - // Translate address if necessary - let (desc_addr, iommu_mapping_cb) = - if let Some(iommu_mapping_cb) = self.iommu_mapping_cb.clone() { - ( - (iommu_mapping_cb)(desc.addr).unwrap(), - Some(iommu_mapping_cb), - ) - } else { - (desc.addr, None) - }; - - let chain = DescriptorChain { - mem: self.mem, - desc_table: self.addr, - table_size: (self.len / 16).try_into().unwrap(), - ttl: (self.len / 16).try_into().unwrap(), - index: 0, - addr: GuestAddress(desc_addr), - len: desc.len, - flags: desc.flags, - next: desc.next, - iommu_mapping_cb, - }; - - if !chain.is_valid() { - return Err(Error::InvalidChain); - } - - Ok(chain) - } - - /// Returns a copy of a descriptor referencing a different GuestMemoryMmap object. - pub fn new_from_head( - mem: &'a GuestMemoryMmap, - head: DescriptorHead, - ) -> Result, Error> { - match DescriptorChain::checked_new( - mem, - head.desc_table, - head.table_size, - head.index, - head.iommu_mapping_cb, - ) { - Some(d) => Ok(d), - None => Err(Error::InvalidChain), - } - } - - /// Returns a DescriptorHead that can be used to build a copy of a descriptor - /// referencing a different GuestMemoryMmap. - pub fn get_head(&self) -> DescriptorHead { - DescriptorHead { - desc_table: self.desc_table, - table_size: self.table_size, - index: self.index, - iommu_mapping_cb: self.iommu_mapping_cb.clone(), - } - } - - fn is_valid(&self) -> bool { - !(!self.mem.check_range(self.addr, self.len as usize) - || (self.has_next() && self.next >= self.table_size)) - } - - /// Gets if this descriptor has another descriptor linked after it. - pub fn has_next(&self) -> bool { - self.flags & VIRTQ_DESC_F_NEXT != 0 && self.ttl > 1 - } - - /// If the driver designated this as a write only descriptor. - /// - /// If this is false, this descriptor is read only. - /// Write only means that the emulated device can write and the driver can read. - pub fn is_write_only(&self) -> bool { - self.flags & VIRTQ_DESC_F_WRITE != 0 - } - - pub fn is_indirect(&self) -> bool { - self.flags & VIRTQ_DESC_F_INDIRECT != 0 - } - - /// Gets the next descriptor in this descriptor chain, if there is one. - /// - /// Note that this is distinct from the next descriptor chain returned by `AvailIter`, which is - /// the head of the next _available_ descriptor chain. - pub fn next_descriptor(&self) -> Option> { - if self.has_next() { - DescriptorChain::checked_new( - self.mem, - self.desc_table, - self.table_size, - self.next, - self.iommu_mapping_cb.clone(), - ) - .map(|mut c| { - c.ttl = self.ttl - 1; - c - }) - } else { - None - } - } -} - -impl<'a> IntoIterator for DescriptorChain<'a> { - type Item = DescriptorChain<'a>; - type IntoIter = DescIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - DescIter { next: Some(self) } - } -} - -/// Consuming iterator over all available descriptor chain heads in the queue. -pub struct AvailIter<'a, 'b> { - mem: &'a GuestMemoryMmap, - desc_table: GuestAddress, - avail_ring: GuestAddress, - next_index: Wrapping, - last_index: Wrapping, - queue_size: u16, - next_avail: &'b mut Wrapping, - iommu_mapping_cb: Option>, -} - -impl<'a, 'b> AvailIter<'a, 'b> { - pub fn new(mem: &'a GuestMemoryMmap, q_next_avail: &'b mut Wrapping) -> AvailIter<'a, 'b> { - AvailIter { - mem, - desc_table: GuestAddress(0), - avail_ring: GuestAddress(0), - next_index: Wrapping(0), - last_index: Wrapping(0), - queue_size: 0, - next_avail: q_next_avail, - iommu_mapping_cb: None, - } - } -} - -impl<'a, 'b> Iterator for AvailIter<'a, 'b> { - type Item = DescriptorChain<'a>; - - fn next(&mut self) -> Option { - if self.next_index == self.last_index { - return None; - } - - let offset = (4 + (self.next_index.0 % self.queue_size) * 2) as usize; - let avail_addr = match self.mem.checked_offset(self.avail_ring, offset) { - Some(a) => a, - None => return None, - }; - // This index is checked below in checked_new - let desc_index: u16 = match self.mem.read_obj(avail_addr) { - Ok(ret) => ret, - Err(_) => { - // TODO log address - error!("Failed to read from memory"); - return None; - } - }; - - self.next_index += Wrapping(1); - - let ret = DescriptorChain::checked_new( - self.mem, - self.desc_table, - self.queue_size, - desc_index, - self.iommu_mapping_cb.clone(), - ); - if ret.is_some() { - *self.next_avail += Wrapping(1); - } - ret - } -} - -#[derive(Clone)] -/// A virtio queue's parameters. -pub struct Queue { - /// The maximal size in elements offered by the device - pub max_size: u16, - - /// The queue size in elements the driver selected - pub size: u16, - - /// Indicates if the queue is finished with configuration - pub ready: bool, - - /// Interrupt vector index of the queue - pub vector: u16, - - /// Guest physical address of the descriptor table - pub desc_table: GuestAddress, - - /// Guest physical address of the available ring - pub avail_ring: GuestAddress, - - /// Guest physical address of the used ring - pub used_ring: GuestAddress, - - pub next_avail: Wrapping, - pub next_used: Wrapping, - - pub iommu_mapping_cb: Option>, - - /// VIRTIO_F_RING_EVENT_IDX negotiated - event_idx: bool, - - /// The last used value when using EVENT_IDX - signalled_used: Option>, -} - -impl Queue { - /// Constructs an empty virtio queue with the given `max_size`. - pub fn new(max_size: u16) -> Queue { - Queue { - max_size, - size: max_size, - ready: false, - vector: VIRTIO_MSI_NO_VECTOR, - desc_table: GuestAddress(0), - avail_ring: GuestAddress(0), - used_ring: GuestAddress(0), - next_avail: Wrapping(0), - next_used: Wrapping(0), - iommu_mapping_cb: None, - event_idx: false, - signalled_used: None, - } - } - - pub fn get_max_size(&self) -> u16 { - self.max_size - } - - pub fn enable(&mut self, set: bool) { - self.ready = set; - - if set { - // Translate address of descriptor table and vrings. - if let Some(iommu_mapping_cb) = &self.iommu_mapping_cb { - self.desc_table = - GuestAddress((iommu_mapping_cb)(self.desc_table.raw_value()).unwrap()); - self.avail_ring = - GuestAddress((iommu_mapping_cb)(self.avail_ring.raw_value()).unwrap()); - self.used_ring = - GuestAddress((iommu_mapping_cb)(self.used_ring.raw_value()).unwrap()); - } - } else { - self.desc_table = GuestAddress(0); - self.avail_ring = GuestAddress(0); - self.used_ring = GuestAddress(0); - } - } - - /// Return the actual size of the queue, as the driver may not set up a - /// queue as big as the device allows. - pub fn actual_size(&self) -> u16 { - min(self.size, self.max_size) - } - - /// Reset the queue to a state that is acceptable for a device reset - pub fn reset(&mut self) { - self.ready = false; - self.size = self.max_size; - self.next_avail = Wrapping(0); - self.next_used = Wrapping(0); - self.vector = VIRTIO_MSI_NO_VECTOR; - self.desc_table = GuestAddress(0); - self.avail_ring = GuestAddress(0); - self.used_ring = GuestAddress(0); - self.event_idx = false; - self.signalled_used = None; - } - - pub fn is_valid(&self, mem: &GuestMemoryMmap) -> bool { - let queue_size = self.actual_size() as usize; - let desc_table = self.desc_table; - let desc_table_size = 16 * queue_size; - let avail_ring = self.avail_ring; - let avail_ring_size = 6 + 2 * queue_size; - let used_ring = self.used_ring; - let used_ring_size = 6 + 8 * queue_size; - if !self.ready { - error!("attempt to use virtio queue that is not marked ready"); - false - } else if self.size > self.max_size || self.size == 0 || (self.size & (self.size - 1)) != 0 - { - error!("virtio queue with invalid size: {}", self.size); - false - } else if desc_table - .checked_add(desc_table_size as GuestUsize) - .map_or(true, |v| !mem.address_in_range(v)) - { - error!( - "virtio queue descriptor table goes out of bounds: start:0x{:08x} size:0x{:08x}", - desc_table.raw_value(), - desc_table_size - ); - false - } else if avail_ring - .checked_add(avail_ring_size as GuestUsize) - .map_or(true, |v| !mem.address_in_range(v)) - { - error!( - "virtio queue available ring goes out of bounds: start:0x{:08x} size:0x{:08x}", - avail_ring.raw_value(), - avail_ring_size - ); - false - } else if used_ring - .checked_add(used_ring_size as GuestUsize) - .map_or(true, |v| !mem.address_in_range(v)) - { - error!( - "virtio queue used ring goes out of bounds: start:0x{:08x} size:0x{:08x}", - used_ring.raw_value(), - used_ring_size - ); - false - } else if desc_table.mask(0xf) != 0 { - error!("virtio queue descriptor table breaks alignment constraints"); - false - } else if avail_ring.mask(0x1) != 0 { - error!("virtio queue available ring breaks alignment constraints"); - false - } else if used_ring.mask(0x3) != 0 { - error!("virtio queue used ring breaks alignment constraints"); - false - } else { - true - } - } - - /// A consuming iterator over all available descriptor chain heads offered by the driver. - pub fn iter<'a, 'b>(&'b mut self, mem: &'a GuestMemoryMmap) -> AvailIter<'a, 'b> { - let queue_size = self.actual_size(); - let avail_ring = self.avail_ring; - - let index_addr = match mem.checked_offset(avail_ring, 2) { - Some(ret) => ret, - None => { - // TODO log address - warn!("Invalid offset"); - return AvailIter::new(mem, &mut self.next_avail); - } - }; - // Note that last_index has no invalid values - let last_index: u16 = match mem.read_obj::(index_addr) { - Ok(ret) => ret, - Err(_) => return AvailIter::new(mem, &mut self.next_avail), - }; - - AvailIter { - mem, - desc_table: self.desc_table, - avail_ring, - next_index: self.next_avail, - last_index: Wrapping(last_index), - queue_size, - next_avail: &mut self.next_avail, - iommu_mapping_cb: self.iommu_mapping_cb.clone(), - } - } - - /// Update avail_event on the used ring with the last index in the avail ring. - pub fn update_avail_event(&mut self, mem: &GuestMemoryMmap) { - let index_addr = match mem.checked_offset(self.avail_ring, 2) { - Some(ret) => ret, - None => { - // TODO log address - warn!("Invalid offset"); - return; - } - }; - // Note that last_index has no invalid values - let last_index: u16 = match mem.read_obj::(index_addr) { - Ok(ret) => ret, - Err(_) => return, - }; - - match mem.checked_offset(self.used_ring, (4 + self.actual_size() * 8) as usize) { - Some(a) => { - mem.write_obj(last_index, a).unwrap(); - } - None => warn!("Can't update avail_event"), - } - - // This fence ensures both guest and us see the correct value (avail idx and avail event) - fence(Ordering::SeqCst); - } - - /// Return the value present in the used_event field of the avail ring. - #[inline(always)] - pub fn get_used_event(&self, mem: &GuestMemoryMmap) -> Option> { - let avail_ring = self.avail_ring; - let used_event_addr = - match mem.checked_offset(avail_ring, (4 + self.actual_size() * 2) as usize) { - Some(a) => a, - None => { - warn!("Invalid offset looking for used_event"); - return None; - } - }; - - // This fence ensures we're seeing the latest update from the guest. - fence(Ordering::SeqCst); - match mem.read_obj::(used_event_addr) { - Ok(ret) => Some(Wrapping(ret)), - Err(_) => None, - } - } - - /// Puts an available descriptor head into the used ring for use by the guest. - pub fn add_used(&mut self, mem: &GuestMemoryMmap, desc_index: u16, len: u32) -> Option { - if desc_index >= self.actual_size() { - error!( - "attempted to add out of bounds descriptor to used ring: {}", - desc_index - ); - return None; - } - - let used_ring = self.used_ring; - let next_used = u64::from(self.next_used.0 % self.actual_size()); - let used_elem = used_ring.unchecked_add(4 + next_used * 8); - - // These writes can't fail as we are guaranteed to be within the descriptor ring. - mem.write_obj(u32::from(desc_index), used_elem).unwrap(); - mem.write_obj(len as u32, used_elem.unchecked_add(4)) - .unwrap(); - - self.next_used += Wrapping(1); - - // This fence ensures all descriptor writes are visible before the index update is. - fence(Ordering::Release); - - mem.write_obj(self.next_used.0 as u16, used_ring.unchecked_add(2)) - .unwrap(); - - Some(self.next_used.0) - } - - /// Goes back one position in the available descriptor chain offered by the driver. - /// Rust does not support bidirectional iterators. This is the only way to revert the effect - /// of an iterator increment on the queue. - pub fn go_to_previous_position(&mut self) { - self.next_avail -= Wrapping(1); - } - - /// Get ring's index from memory. - fn index_from_memory(&self, ring: GuestAddress, mem: &GuestMemoryMmap) -> Result { - mem.read_obj::( - mem.checked_offset(ring, 2) - .ok_or_else(|| Error::InvalidOffset(ring.raw_value() + 2))?, - ) - .map_err(Error::InvalidRingIndexFromMemory) - } - - /// Get latest index from available ring. - pub fn avail_index_from_memory(&self, mem: &GuestMemoryMmap) -> Result { - self.index_from_memory(self.avail_ring, mem) - } - - /// Get latest index from used ring. - pub fn used_index_from_memory(&self, mem: &GuestMemoryMmap) -> Result { - self.index_from_memory(self.used_ring, mem) - } - - pub fn available_descriptors(&self, mem: &GuestMemoryMmap) -> Result { - Ok(self.used_index_from_memory(mem)? < self.avail_index_from_memory(mem)?) - } - - pub fn set_event_idx(&mut self, enabled: bool) { - /* Also reset the last signalled event */ - self.signalled_used = None; - self.event_idx = enabled; - } - - pub fn needs_notification(&mut self, mem: &GuestMemoryMmap, used_idx: Wrapping) -> bool { - if !self.event_idx { - return true; - } - - let mut notify = true; - - if let Some(old_idx) = self.signalled_used { - if let Some(used_event) = self.get_used_event(mem) { - debug!( - "used_event = {:?} used_idx = {:?} old_idx = {:?}", - used_event, used_idx, old_idx - ); - if (used_idx - used_event - Wrapping(1u16)) >= (used_idx - old_idx) { - notify = false; - } - } - } - - self.signalled_used = Some(used_idx); - debug!("Needs notification: {:?}", notify); - notify - } -} - pub mod testing { - use super::*; use std::marker::PhantomData; use std::mem; - use vm_memory::Bytes; + use virtio_queue::{Queue, QueueState}; use vm_memory::{bitmap::AtomicBitmap, Address, GuestAddress, GuestUsize}; + use vm_memory::{Bytes, GuestMemoryAtomic}; type GuestMemoryMmap = vm_memory::GuestMemoryMmap; @@ -885,6 +179,7 @@ pub mod testing { pub dtable: Vec>, pub avail: VirtqAvail<'a>, pub used: VirtqUsed<'a>, + pub mem: &'a GuestMemoryMmap, } impl<'a> VirtQueue<'a> { @@ -918,6 +213,7 @@ pub mod testing { dtable, avail, used, + mem, } } @@ -938,14 +234,18 @@ pub mod testing { } // Creates a new Queue, using the underlying memory regions represented by the VirtQueue. - pub fn create_queue(&self) -> Queue { - let mut q = Queue::new(self.size()); + pub fn create_queue(&self) -> Queue> { + let mem = GuestMemoryAtomic::new(self.mem.clone()); + let mut q = Queue::< + GuestMemoryAtomic, + QueueState>, + >::new(mem, self.size()); - q.size = self.size(); - q.ready = true; - q.desc_table = self.dtable_start(); - q.avail_ring = self.avail_start(); - q.used_ring = self.used_start(); + q.state.size = self.size(); + q.state.ready = true; + q.state.desc_table = self.dtable_start(); + q.state.avail_ring = self.avail_start(); + q.state.used_ring = self.used_start(); q } @@ -959,229 +259,3 @@ pub mod testing { } } } - -#[cfg(test)] -pub mod tests { - use super::testing::*; - pub use super::*; - use vm_memory::{bitmap::AtomicBitmap, GuestAddress}; - - type GuestMemoryMmap = vm_memory::GuestMemoryMmap; - - #[test] - fn test_checked_new_descriptor_chain() { - let m = &GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap(); - let vq = VirtQueue::new(GuestAddress(0), m, 16); - - assert!(vq.end().0 < 0x1000); - - // index >= queue_size - assert!(DescriptorChain::checked_new(m, vq.start(), 16, 16, None).is_none()); - - // desc_table address is way off - assert!( - DescriptorChain::checked_new(m, GuestAddress(0x00ff_ffff_ffff), 16, 0, None).is_none() - ); - - // the addr field of the descriptor is way off - vq.dtable[0].addr.set(0x0fff_ffff_ffff); - assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0, None).is_none()); - - // let's create some invalid chains - - { - // the addr field of the desc is ok now - vq.dtable[0].addr.set(0x1000); - // ...but the length is too large - vq.dtable[0].len.set(0xffff_ffff); - assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0, None).is_none()); - } - - { - // the first desc has a normal len now, and the next_descriptor flag is set - vq.dtable[0].len.set(0x1000); - vq.dtable[0].flags.set(VIRTQ_DESC_F_NEXT); - //..but the index of the next descriptor is too large - vq.dtable[0].next.set(16); - - assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0, None).is_none()); - } - - // finally, let's test an ok chain - - { - vq.dtable[0].next.set(1); - vq.dtable[1].set(0x2000, 0x1000, 0, 0); - - let c = DescriptorChain::checked_new(m, vq.start(), 16, 0, None).unwrap(); - - assert_eq!(c.mem as *const GuestMemoryMmap, m as *const GuestMemoryMmap); - assert_eq!(c.desc_table, vq.start()); - assert_eq!(c.table_size, 16); - assert_eq!(c.ttl, c.table_size); - assert_eq!(c.index, 0); - assert_eq!(c.addr, GuestAddress(0x1000)); - assert_eq!(c.len, 0x1000); - assert_eq!(c.flags, VIRTQ_DESC_F_NEXT); - assert_eq!(c.next, 1); - - assert!(c.next_descriptor().unwrap().next_descriptor().is_none()); - } - } - - #[test] - fn test_new_from_descriptor_chain() { - let m = &GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap(); - let vq = VirtQueue::new(GuestAddress(0), m, 16); - - // create a chain with a descriptor pointing to an indirect table - vq.dtable[0].addr.set(0x1000); - vq.dtable[0].len.set(0x1000); - vq.dtable[0].next.set(0); - vq.dtable[0].flags.set(VIRTQ_DESC_F_INDIRECT); - - let c = DescriptorChain::checked_new(m, vq.start(), 16, 0, None).unwrap(); - assert!(c.is_indirect()); - - // create an indirect table with 4 chained descriptors - let mut indirect_table = Vec::with_capacity(4); - for j in 0..4 { - let desc = VirtqDesc::new(GuestAddress(0x1000 + (j * 16)), m); - desc.set(0x1000, 0x1000, VIRTQ_DESC_F_NEXT, (j + 1) as u16); - indirect_table.push(desc); - } - - // try to iterate through the indirect table descriptors - let mut i = c.new_from_indirect().unwrap(); - for j in 0..4 { - assert_eq!(i.flags, VIRTQ_DESC_F_NEXT); - assert_eq!(i.next, j + 1); - i = i.next_descriptor().unwrap(); - } - } - - #[test] - fn test_queue_and_iterator() { - let m = &GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap(); - let vq = VirtQueue::new(GuestAddress(0), m, 16); - - let mut q = vq.create_queue(); - - // q is currently valid - assert!(q.is_valid(m)); - - // shouldn't be valid when not marked as ready - q.ready = false; - assert!(!q.is_valid(m)); - q.ready = true; - - // or when size > max_size - q.size = q.max_size << 1; - assert!(!q.is_valid(m)); - q.size = q.max_size; - - // or when size is 0 - q.size = 0; - assert!(!q.is_valid(m)); - q.size = q.max_size; - - // or when size is not a power of 2 - q.size = 11; - assert!(!q.is_valid(m)); - q.size = q.max_size; - - // or if the various addresses are off - - q.desc_table = GuestAddress(0xffff_ffff); - assert!(!q.is_valid(m)); - q.desc_table = GuestAddress(0x1001); - assert!(!q.is_valid(m)); - q.desc_table = vq.dtable_start(); - - q.avail_ring = GuestAddress(0xffff_ffff); - assert!(!q.is_valid(m)); - q.avail_ring = GuestAddress(0x1001); - assert!(!q.is_valid(m)); - q.avail_ring = vq.avail_start(); - - q.used_ring = GuestAddress(0xffff_ffff); - assert!(!q.is_valid(m)); - q.used_ring = GuestAddress(0x1001); - assert!(!q.is_valid(m)); - q.used_ring = vq.used_start(); - - { - // an invalid queue should return an iterator with no next - q.ready = false; - let mut i = q.iter(m); - assert!(i.next().is_none()); - } - - q.ready = true; - - // now let's create two simple descriptor chains - - { - for j in 0..5 { - vq.dtable[j].set( - 0x1000 * (j + 1) as u64, - 0x1000, - VIRTQ_DESC_F_NEXT, - (j + 1) as u16, - ); - } - - // the chains are (0, 1) and (2, 3, 4) - vq.dtable[1].flags.set(0); - vq.dtable[4].flags.set(0); - vq.avail.ring[0].set(0); - vq.avail.ring[1].set(2); - vq.avail.idx.set(2); - - let mut i = q.iter(m); - - { - let mut c = i.next().unwrap(); - c = c.next_descriptor().unwrap(); - assert!(!c.has_next()); - } - - { - let mut c = i.next().unwrap(); - c = c.next_descriptor().unwrap(); - c = c.next_descriptor().unwrap(); - assert!(!c.has_next()); - } - } - - // also test go_to_previous_position() works as expected - { - assert!(q.iter(m).next().is_none()); - q.go_to_previous_position(); - let mut c = q.iter(m).next().unwrap(); - c = c.next_descriptor().unwrap(); - c = c.next_descriptor().unwrap(); - assert!(!c.has_next()); - } - } - - #[test] - fn test_add_used() { - let m = &GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap(); - let vq = VirtQueue::new(GuestAddress(0), m, 16); - - let mut q = vq.create_queue(); - assert_eq!(vq.used.idx.get(), 0); - - //index too large - q.add_used(m, 16, 0x1000); - assert_eq!(vq.used.idx.get(), 0); - - //should be ok - q.add_used(m, 1, 0x1000); - assert_eq!(vq.used.idx.get(), 1); - let x = vq.used.ring[0].get(); - assert_eq!(x.id, 1); - assert_eq!(x.len, 0x1000); - } -} diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index a12eff9db..d05900d45 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -48,6 +48,7 @@ vfio-ioctls = { git = "https://github.com/rust-vmm/vfio-ioctls", branch = "main" vfio_user = { path = "../vfio_user" } vhdx = { path = "../vhdx" } virtio-devices = { path = "../virtio-devices" } +virtio-queue = { path = "../virtio-queue" } vm-allocator = { path = "../vm-allocator" } vm-device = { path = "../vm-device" } vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] } diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index 5525b31e8..1ebce16fe 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -91,9 +91,10 @@ use vfio_ioctls::{VfioContainer, VfioDevice}; use virtio_devices::transport::VirtioPciDevice; use virtio_devices::transport::VirtioTransport; use virtio_devices::vhost_user::VhostUserConfig; -use virtio_devices::VirtioMemMappingSource; -use virtio_devices::{DmaRemapping, Endpoint, IommuMapping}; +use virtio_devices::{AccessPlatformMapping, VirtioMemMappingSource}; +use virtio_devices::{Endpoint, IommuMapping}; use virtio_devices::{VirtioSharedMemory, VirtioSharedMemoryList}; +use virtio_queue::AccessPlatform; use vm_allocator::SystemAllocator; use vm_device::dma_mapping::vfio::VfioDmaMapping; use vm_device::interrupt::{ @@ -109,7 +110,7 @@ use vm_migration::{ protocol::MemoryRangeTable, Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable, Transportable, }; -use vm_virtio::{VirtioDeviceType, VirtioIommuRemapping}; +use vm_virtio::VirtioDeviceType; use vmm_sys_util::eventfd::EventFd; #[cfg(target_arch = "aarch64")] @@ -3165,26 +3166,18 @@ impl DeviceManager { // about a virtio config change. let msix_num = (virtio_device.lock().unwrap().queue_max_sizes().len() + 1) as u16; - // Create the callback from the implementation of the DmaRemapping - // trait. The point with the callback is to simplify the code as we - // know about the device ID from this point. - let iommu_mapping_cb: Option> = - if let Some(mapping) = iommu_mapping { - let mapping_clone = mapping.clone(); - Some(Arc::new(Box::new(move |addr: u64| { - mapping_clone.translate(pci_device_bdf, addr).map_err(|e| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!( - "failed to translate addr 0x{:x} for device 00:{:02x}.0 {}", - addr, pci_device_bdf, e - ), - ) - }) - }) as VirtioIommuRemapping)) - } else { - None - }; + // Create the AccessPlatform trait from the implementation IommuMapping. + // This will provide address translation for any virtio device sitting + // behind a vIOMMU. + let access_platform: Option> = if let Some(mapping) = iommu_mapping + { + Some(Arc::new(AccessPlatformMapping::new( + pci_device_bdf, + mapping.clone(), + ))) + } else { + None + }; let memory = self.memory_manager.lock().unwrap().guest_memory(); let mut virtio_pci_device = VirtioPciDevice::new( @@ -3192,7 +3185,7 @@ impl DeviceManager { memory, virtio_device, msix_num, - iommu_mapping_cb, + access_platform, &self.msi_interrupt_manager, pci_device_bdf, self.activate_evt