From f87156b77662af63b9fc9f116a213dd69e16007b Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Thu, 6 Oct 2022 11:59:26 +0200 Subject: [PATCH] vhost_kern: vdpa: Add missing ioctls New ioctls have been recently introduced to interact with vDPA devices. This patch is based on Linux kernel v6.0, adding the list of missing ioctls: - VHOST_VDPA_GET_CONFIG_SIZE - VHOST_VDPA_GET_VQS_COUNT - VHOST_VDPA_GET_GROUP_NUM - VHOST_VDPA_GET_AS_NUM - VHOST_VDPA_GET_VRING_GROUP - VHOST_VDPA_SET_GROUP_ASID - VHOST_VDPA_SUSPEND Signed-off-by: Sebastien Boeuf --- src/vdpa.rs | 31 ++++++++++ src/vhost_kern/vdpa.rs | 101 +++++++++++++++++++++++++++++--- src/vhost_kern/vhost_binding.rs | 9 +++ 3 files changed, 134 insertions(+), 7 deletions(-) diff --git a/src/vdpa.rs b/src/vdpa.rs index a55ab92..6af01cf 100644 --- a/src/vdpa.rs +++ b/src/vdpa.rs @@ -77,6 +77,37 @@ pub trait VhostVdpa: VhostBackend { /// Get the valid I/O virtual addresses range supported by the device. fn get_iova_range(&self) -> Result; + /// Get the config size + fn get_config_size(&self) -> Result; + + /// Get the count of all virtqueues + fn get_vqs_count(&self) -> Result; + + /// Get the number of virtqueue groups + fn get_group_num(&self) -> Result; + + /// Get the number of address spaces + fn get_as_num(&self) -> Result; + + /// Get the group for a virtqueue. + /// The virtqueue index is stored in the index field of + /// vhost_vring_state. The group for this specific virtqueue is + /// returned via num field of vhost_vring_state. + fn get_vring_group(&self, queue_index: u32) -> Result; + + /// Set the ASID for a virtqueue group. The group index is stored in + /// the index field of vhost_vring_state, the ASID associated with this + /// group is stored at num field of vhost_vring_state. + fn set_group_asid(&self, group_index: u32, asid: u32) -> Result<()>; + + /// Suspend a device so it does not process virtqueue requests anymore + /// + /// After the return of ioctl the device must preserve all the necessary state + /// (the virtqueue vring base plus the possible device specific states) that is + /// required for restoring in the future. The device must not change its + /// configuration after that point. + fn suspend(&self) -> Result<()>; + /// Map DMA region. /// /// # Arguments diff --git a/src/vhost_kern/vdpa.rs b/src/vhost_kern/vdpa.rs index cd61c17..39929c3 100644 --- a/src/vhost_kern/vdpa.rs +++ b/src/vhost_kern/vdpa.rs @@ -12,7 +12,7 @@ use std::os::unix::io::{AsRawFd, RawFd}; use vm_memory::GuestAddressSpace; use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::fam::*; -use vmm_sys_util::ioctl::{ioctl_with_mut_ref, ioctl_with_ptr, ioctl_with_ref}; +use vmm_sys_util::ioctl::{ioctl, ioctl_with_mut_ref, ioctl_with_ptr, ioctl_with_ref}; use super::vhost_binding::*; use super::{ioctl_result, Error, Result, VhostKernBackend, VhostKernFeatures}; @@ -149,6 +149,57 @@ impl VhostVdpa for VhostKernVdpa { ioctl_result(ret, iova_range) } + fn get_config_size(&self) -> Result { + let mut config_size: u32 = 0; + let ret = + unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_CONFIG_SIZE(), &mut config_size) }; + ioctl_result(ret, config_size) + } + + fn get_vqs_count(&self) -> Result { + let mut vqs_count: u32 = 0; + let ret = unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_VQS_COUNT(), &mut vqs_count) }; + ioctl_result(ret, vqs_count) + } + + fn get_group_num(&self) -> Result { + let mut group_num: u32 = 0; + let ret = unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_GROUP_NUM(), &mut group_num) }; + ioctl_result(ret, group_num) + } + + fn get_as_num(&self) -> Result { + let mut as_num: u32 = 0; + let ret = unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_AS_NUM(), &mut as_num) }; + ioctl_result(ret, as_num) + } + + fn get_vring_group(&self, queue_index: u32) -> Result { + let mut vring_state = vhost_vring_state { + index: queue_index, + ..Default::default() + }; + + let ret = + unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_VRING_GROUP(), &mut vring_state) }; + ioctl_result(ret, vring_state.num) + } + + fn set_group_asid(&self, group_index: u32, asid: u32) -> Result<()> { + let vring_state = vhost_vring_state { + index: group_index, + num: asid, + }; + + let ret = unsafe { ioctl_with_ref(self, VHOST_VDPA_GET_VRING_GROUP(), &vring_state) }; + ioctl_result(ret, ()) + } + + fn suspend(&self) -> Result<()> { + let ret = unsafe { ioctl(self, VHOST_VDPA_SUSPEND()) }; + ioctl_result(ret, ()) + } + fn dma_map(&self, iova: u64, size: u64, vaddr: *const u8, readonly: bool) -> Result<()> { let iotlb = VhostIotlbMsg { iova, @@ -253,6 +304,20 @@ mod tests { }; } + macro_rules! validate_ioctl { + ( $e:expr, $ref_value:expr ) => { + match $e { + Ok(v) => assert_eq!(v, $ref_value), + Err(error) => match error { + Error::IoctlError(e) if e.raw_os_error().unwrap() == libc::ENOTTY => { + println!("Err: {:?} SKIPPED", e); + } + e => panic!("Err: {:?}", e), + }, + } + }; + } + #[test] #[serial] fn test_vdpa_kern_new_device() { @@ -313,7 +378,8 @@ mod tests { }; vdpa.set_mem_table(&[region]).unwrap(); - assert!(vdpa.get_device_id().unwrap() > 0); + let device_id = vdpa.get_device_id().unwrap(); + assert!(device_id > 0); assert_eq!(vdpa.get_status().unwrap(), 0x0); vdpa.set_status(0x1).unwrap(); @@ -360,6 +426,32 @@ mod tests { vdpa.set_config_call(&eventfd).unwrap(); + let iova_range = vdpa.get_iova_range().unwrap(); + // vDPA-block simulator returns [0, u64::MAX] range + assert_eq!(iova_range.first, 0); + assert_eq!(iova_range.last, u64::MAX); + + let (config_size, vqs_count, group_num, as_num, vring_group) = if device_id == 1 { + (24, 3, 2, 2, 0) + } else if device_id == 2 { + (60, 1, 1, 1, 0) + } else { + panic!("Unexpected device id {}", device_id) + }; + + validate_ioctl!(vdpa.get_config_size(), config_size); + validate_ioctl!(vdpa.get_vqs_count(), vqs_count); + validate_ioctl!(vdpa.get_group_num(), group_num); + validate_ioctl!(vdpa.get_as_num(), as_num); + validate_ioctl!(vdpa.get_vring_group(0), vring_group); + validate_ioctl!(vdpa.set_group_asid(0, 12345), ()); + + if vdpa.get_backend_features().unwrap() & (1 << VHOST_BACKEND_F_SUSPEND) + == (1 << VHOST_BACKEND_F_SUSPEND) + { + validate_ioctl!(vdpa.suspend(), ()); + } + assert_eq!(vdpa.get_vring_base(0).unwrap(), 1); vdpa.set_vring_enable(0, true).unwrap(); @@ -383,11 +475,6 @@ mod tests { vdpa.set_owner().unwrap(); - let iova_range = vdpa.get_iova_range().unwrap(); - // vDPA-block simulator returns [0, u64::MAX] range - assert_eq!(iova_range.first, 0); - assert_eq!(iova_range.last, u64::MAX); - vdpa.dma_map(0xFFFF_0000, 0xFFFF, std::ptr::null::(), false) .unwrap_err(); diff --git a/src/vhost_kern/vhost_binding.rs b/src/vhost_kern/vhost_binding.rs index 2b0d974..a4fee8d 100644 --- a/src/vhost_kern/vhost_binding.rs +++ b/src/vhost_kern/vhost_binding.rs @@ -40,6 +40,8 @@ pub const VHOST_NET_F_VIRTIO_NET_HDR: raw::c_uint = 27; pub const VHOST_SCSI_ABI_VERSION: raw::c_uint = 1; pub const VHOST_BACKEND_F_IOTLB_MSG_V2: raw::c_ulonglong = 0x1; pub const VHOST_BACKEND_F_IOTLB_BATCH: raw::c_ulonglong = 0x2; +pub const VHOST_BACKEND_F_IOTLB_ASID: raw::c_ulonglong = 0x3; +pub const VHOST_BACKEND_F_SUSPEND: raw::c_ulonglong = 0x4; ioctl_ior_nr!(VHOST_GET_FEATURES, VHOST, 0x00, raw::c_ulonglong); ioctl_iow_nr!(VHOST_SET_FEATURES, VHOST, 0x00, raw::c_ulonglong); @@ -79,6 +81,13 @@ ioctl_ior_nr!( 0x78, vhost_vdpa_iova_range ); +ioctl_ior_nr!(VHOST_VDPA_GET_CONFIG_SIZE, VHOST, 0x79, raw::c_uint); +ioctl_ior_nr!(VHOST_VDPA_GET_VQS_COUNT, VHOST, 0x80, raw::c_uint); +ioctl_ior_nr!(VHOST_VDPA_GET_GROUP_NUM, VHOST, 0x81, raw::c_uint); +ioctl_ior_nr!(VHOST_VDPA_GET_AS_NUM, VHOST, 0x7a, raw::c_uint); +ioctl_iowr_nr!(VHOST_VDPA_GET_VRING_GROUP, VHOST, 0x7b, vhost_vring_state); +ioctl_iow_nr!(VHOST_VDPA_SET_GROUP_ASID, VHOST, 0x7c, vhost_vring_state); +ioctl_io_nr!(VHOST_VDPA_SUSPEND, VHOST, 0x7d); #[repr(C)] #[derive(Default)]