vhost_kern: vdpa: Add missing ioctls

New ioctls have been recently introduced to interact with vDPA devices.

This patch is based on Linux kernel v6.0, adding the list of missing
ioctls:

- VHOST_VDPA_GET_CONFIG_SIZE
- VHOST_VDPA_GET_VQS_COUNT
- VHOST_VDPA_GET_GROUP_NUM
- VHOST_VDPA_GET_AS_NUM
- VHOST_VDPA_GET_VRING_GROUP
- VHOST_VDPA_SET_GROUP_ASID
- VHOST_VDPA_SUSPEND

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

src/vdpa.rs

@@ -77,6 +77,37 @@ pub trait VhostVdpa: VhostBackend {
     /// Get the valid I/O virtual addresses range supported by the device.
     fn get_iova_range(&self) -> Result<VhostVdpaIovaRange>;
 
+    /// Get the config size
+    fn get_config_size(&self) -> Result<u32>;
+
+    /// Get the count of all virtqueues
+    fn get_vqs_count(&self) -> Result<u32>;
+
+    /// Get the number of virtqueue groups
+    fn get_group_num(&self) -> Result<u32>;
+
+    /// Get the number of address spaces
+    fn get_as_num(&self) -> Result<u32>;
+
+    /// Get the group for a virtqueue.
+    /// The virtqueue index is stored in the index field of
+    /// vhost_vring_state. The group for this specific virtqueue is
+    /// returned via num field of vhost_vring_state.
+    fn get_vring_group(&self, queue_index: u32) -> Result<u32>;
+
+    /// Set the ASID for a virtqueue group. The group index is stored in
+    /// the index field of vhost_vring_state, the ASID associated with this
+    /// group is stored at num field of vhost_vring_state.
+    fn set_group_asid(&self, group_index: u32, asid: u32) -> Result<()>;
+
+    /// Suspend a device so it does not process virtqueue requests anymore
+    ///
+    /// After the return of ioctl the device must preserve all the necessary state
+    /// (the virtqueue vring base plus the possible device specific states) that is
+    /// required for restoring in the future. The device must not change its
+    /// configuration after that point.
+    fn suspend(&self) -> Result<()>;
+
     /// Map DMA region.
     ///
     /// # Arguments

src/vhost_kern/vdpa.rs

@@ -12,7 +12,7 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use vm_memory::GuestAddressSpace;
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::fam::*;
-use vmm_sys_util::ioctl::{ioctl_with_mut_ref, ioctl_with_ptr, ioctl_with_ref};
+use vmm_sys_util::ioctl::{ioctl, ioctl_with_mut_ref, ioctl_with_ptr, ioctl_with_ref};
 
 use super::vhost_binding::*;
 use super::{ioctl_result, Error, Result, VhostKernBackend, VhostKernFeatures};
@@ -149,6 +149,57 @@ impl<AS: GuestAddressSpace> VhostVdpa for VhostKernVdpa<AS> {
         ioctl_result(ret, iova_range)
     }
 
+    fn get_config_size(&self) -> Result<u32> {
+        let mut config_size: u32 = 0;
+        let ret =
+            unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_CONFIG_SIZE(), &mut config_size) };
+        ioctl_result(ret, config_size)
+    }
+
+    fn get_vqs_count(&self) -> Result<u32> {
+        let mut vqs_count: u32 = 0;
+        let ret = unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_VQS_COUNT(), &mut vqs_count) };
+        ioctl_result(ret, vqs_count)
+    }
+
+    fn get_group_num(&self) -> Result<u32> {
+        let mut group_num: u32 = 0;
+        let ret = unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_GROUP_NUM(), &mut group_num) };
+        ioctl_result(ret, group_num)
+    }
+
+    fn get_as_num(&self) -> Result<u32> {
+        let mut as_num: u32 = 0;
+        let ret = unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_AS_NUM(), &mut as_num) };
+        ioctl_result(ret, as_num)
+    }
+
+    fn get_vring_group(&self, queue_index: u32) -> Result<u32> {
+        let mut vring_state = vhost_vring_state {
+            index: queue_index,
+            ..Default::default()
+        };
+
+        let ret =
+            unsafe { ioctl_with_mut_ref(self, VHOST_VDPA_GET_VRING_GROUP(), &mut vring_state) };
+        ioctl_result(ret, vring_state.num)
+    }
+
+    fn set_group_asid(&self, group_index: u32, asid: u32) -> Result<()> {
+        let vring_state = vhost_vring_state {
+            index: group_index,
+            num: asid,
+        };
+
+        let ret = unsafe { ioctl_with_ref(self, VHOST_VDPA_GET_VRING_GROUP(), &vring_state) };
+        ioctl_result(ret, ())
+    }
+
+    fn suspend(&self) -> Result<()> {
+        let ret = unsafe { ioctl(self, VHOST_VDPA_SUSPEND()) };
+        ioctl_result(ret, ())
+    }
+
     fn dma_map(&self, iova: u64, size: u64, vaddr: *const u8, readonly: bool) -> Result<()> {
         let iotlb = VhostIotlbMsg {
             iova,
@@ -253,6 +304,20 @@ mod tests {
         };
     }
 
+    macro_rules! validate_ioctl {
+        ( $e:expr, $ref_value:expr ) => {
+            match $e {
+                Ok(v) => assert_eq!(v, $ref_value),
+                Err(error) => match error {
+                    Error::IoctlError(e) if e.raw_os_error().unwrap() == libc::ENOTTY => {
+                        println!("Err: {:?} SKIPPED", e);
+                    }
+                    e => panic!("Err: {:?}", e),
+                },
+            }
+        };
+    }
+
     #[test]
     #[serial]
     fn test_vdpa_kern_new_device() {
@@ -313,7 +378,8 @@ mod tests {
         };
         vdpa.set_mem_table(&[region]).unwrap();
 
-        assert!(vdpa.get_device_id().unwrap() > 0);
+        let device_id = vdpa.get_device_id().unwrap();
+        assert!(device_id > 0);
 
         assert_eq!(vdpa.get_status().unwrap(), 0x0);
         vdpa.set_status(0x1).unwrap();
@@ -360,6 +426,32 @@ mod tests {
 
         vdpa.set_config_call(&eventfd).unwrap();
 
+        let iova_range = vdpa.get_iova_range().unwrap();
+        // vDPA-block simulator returns [0, u64::MAX] range
+        assert_eq!(iova_range.first, 0);
+        assert_eq!(iova_range.last, u64::MAX);
+
+        let (config_size, vqs_count, group_num, as_num, vring_group) = if device_id == 1 {
+            (24, 3, 2, 2, 0)
+        } else if device_id == 2 {
+            (60, 1, 1, 1, 0)
+        } else {
+            panic!("Unexpected device id {}", device_id)
+        };
+
+        validate_ioctl!(vdpa.get_config_size(), config_size);
+        validate_ioctl!(vdpa.get_vqs_count(), vqs_count);
+        validate_ioctl!(vdpa.get_group_num(), group_num);
+        validate_ioctl!(vdpa.get_as_num(), as_num);
+        validate_ioctl!(vdpa.get_vring_group(0), vring_group);
+        validate_ioctl!(vdpa.set_group_asid(0, 12345), ());
+
+        if vdpa.get_backend_features().unwrap() & (1 << VHOST_BACKEND_F_SUSPEND)
+            == (1 << VHOST_BACKEND_F_SUSPEND)
+        {
+            validate_ioctl!(vdpa.suspend(), ());
+        }
+
         assert_eq!(vdpa.get_vring_base(0).unwrap(), 1);
 
         vdpa.set_vring_enable(0, true).unwrap();
@@ -383,11 +475,6 @@ mod tests {
 
         vdpa.set_owner().unwrap();
 
-        let iova_range = vdpa.get_iova_range().unwrap();
-        // vDPA-block simulator returns [0, u64::MAX] range
-        assert_eq!(iova_range.first, 0);
-        assert_eq!(iova_range.last, u64::MAX);
-
         vdpa.dma_map(0xFFFF_0000, 0xFFFF, std::ptr::null::<u8>(), false)
             .unwrap_err();
 

src/vhost_kern/vhost_binding.rs

@@ -40,6 +40,8 @@ pub const VHOST_NET_F_VIRTIO_NET_HDR: raw::c_uint = 27;
 pub const VHOST_SCSI_ABI_VERSION: raw::c_uint = 1;
 pub const VHOST_BACKEND_F_IOTLB_MSG_V2: raw::c_ulonglong = 0x1;
 pub const VHOST_BACKEND_F_IOTLB_BATCH: raw::c_ulonglong = 0x2;
+pub const VHOST_BACKEND_F_IOTLB_ASID: raw::c_ulonglong = 0x3;
+pub const VHOST_BACKEND_F_SUSPEND: raw::c_ulonglong = 0x4;
 
 ioctl_ior_nr!(VHOST_GET_FEATURES, VHOST, 0x00, raw::c_ulonglong);
 ioctl_iow_nr!(VHOST_SET_FEATURES, VHOST, 0x00, raw::c_ulonglong);
@@ -79,6 +81,13 @@ ioctl_ior_nr!(
     0x78,
     vhost_vdpa_iova_range
 );
+ioctl_ior_nr!(VHOST_VDPA_GET_CONFIG_SIZE, VHOST, 0x79, raw::c_uint);
+ioctl_ior_nr!(VHOST_VDPA_GET_VQS_COUNT, VHOST, 0x80, raw::c_uint);
+ioctl_ior_nr!(VHOST_VDPA_GET_GROUP_NUM, VHOST, 0x81, raw::c_uint);
+ioctl_ior_nr!(VHOST_VDPA_GET_AS_NUM, VHOST, 0x7a, raw::c_uint);
+ioctl_iowr_nr!(VHOST_VDPA_GET_VRING_GROUP, VHOST, 0x7b, vhost_vring_state);
+ioctl_iow_nr!(VHOST_VDPA_SET_GROUP_ASID, VHOST, 0x7c, vhost_vring_state);
+ioctl_io_nr!(VHOST_VDPA_SUSPEND, VHOST, 0x7d);
 
 #[repr(C)]
 #[derive(Default)]