From 253c06d3bac9e02c79082fdf8313000adc69aee7 Mon Sep 17 00:00:00 2001
From: Michael Zhao <michael.zhao@arm.com>
Date: Thu, 12 Aug 2021 21:37:44 +0800
Subject: [PATCH] arch/aarch64: Add virtio-iommu device in FDT

Add a virtio-iommu node into FDT if iommu option is turned on. Now we
support only one virtio-iommu device.

Signed-off-by: Michael Zhao <michael.zhao@arm.com>
---
 arch/src/aarch64/fdt.rs | 48 +++++++++++++++++++++++++++++++++++++++--
 arch/src/aarch64/mod.rs |  2 ++
 vmm/src/vm.rs           | 10 +++++++++
 3 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/arch/src/aarch64/fdt.rs b/arch/src/aarch64/fdt.rs
index 03c9f61c1..1b7165ff9 100644
--- a/arch/src/aarch64/fdt.rs
+++ b/arch/src/aarch64/fdt.rs
@@ -35,9 +35,12 @@ const MSI_PHANDLE: u32 = 2;
 const CLOCK_PHANDLE: u32 = 3;
 // This is a value for uniquely identifying the FDT node containing the gpio controller.
 const GPIO_PHANDLE: u32 = 4;
+// This is a value for virtio-iommu. Now only one virtio-iommu device is supported.
+const VIRTIO_IOMMU_PHANDLE: u32 = 5;
+// NOTE: Keep FIRST_VCPU_PHANDLE the last PHANDLE defined.
 // This is a value for uniquely identifying the FDT node containing the first vCPU.
 // The last number of vCPU phandle depends on the number of vCPUs.
-const FIRST_VCPU_PHANDLE: u32 = 5;
+const FIRST_VCPU_PHANDLE: u32 = 6;
 
 // Read the documentation specified when appending the root node to the FDT.
 const ADDRESS_CELLS: u32 = 0x2;
@@ -87,6 +90,7 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     initrd: &Option<InitramfsConfig>,
     pci_space_address: &(u64, u64),
     numa_nodes: &NumaNodes,
+    virtio_iommu_bdf: Option<u32>,
 ) -> FdtWriterResult<Vec<u8>> {
     // Allocate stuff necessary for the holding the blob.
     let mut fdt = FdtWriter::new().unwrap();
@@ -113,7 +117,12 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     create_clock_node(&mut fdt)?;
     create_psci_node(&mut fdt)?;
     create_devices_node(&mut fdt, device_info)?;
-    create_pci_nodes(&mut fdt, pci_space_address.0, pci_space_address.1)?;
+    create_pci_nodes(
+        &mut fdt,
+        pci_space_address.0,
+        pci_space_address.1,
+        virtio_iommu_bdf,
+    )?;
     if numa_nodes.len() > 1 {
         create_distance_map_node(&mut fdt, numa_nodes)?;
     }
@@ -540,6 +549,7 @@ fn create_pci_nodes(
     fdt: &mut FdtWriter,
     pci_device_base: u64,
     pci_device_size: u64,
+    virtio_iommu_bdf: Option<u32>,
 ) -> FdtWriterResult<()> {
     // Add node for PCIe controller.
     // See Documentation/devicetree/bindings/pci/host-generic-pci.txt in the kernel
@@ -603,6 +613,40 @@ fn create_pci_nodes(
     fdt.property_null("interrupt-map-mask")?;
     fdt.property_null("dma-coherent")?;
     fdt.property_u32("msi-parent", MSI_PHANDLE)?;
+
+    if let Some(virtio_iommu_bdf) = virtio_iommu_bdf {
+        // See kernel document Documentation/devicetree/bindings/pci/pci-iommu.txt
+        // for 'iommu-map' attribute setting.
+        let iommu_map = [
+            0_u32,
+            VIRTIO_IOMMU_PHANDLE,
+            0_u32,
+            virtio_iommu_bdf,
+            virtio_iommu_bdf + 1,
+            VIRTIO_IOMMU_PHANDLE,
+            virtio_iommu_bdf + 1,
+            0xffff - virtio_iommu_bdf,
+        ];
+        fdt.property_array_u32("iommu-map", &iommu_map)?;
+
+        // See kernel document Documentation/devicetree/bindings/virtio/iommu.txt
+        // for virtio-iommu node settings.
+        let virtio_iommu_node_name = format!("virtio_iommu@{:x}", virtio_iommu_bdf);
+        let virtio_iommu_node = fdt.begin_node(&virtio_iommu_node_name)?;
+        fdt.property_u32("#iommu-cells", 1)?;
+        fdt.property_string("compatible", "virtio,pci-iommu")?;
+
+        // 'reg' is a five-cell address encoded as
+        // (phys.hi phys.mid phys.lo size.hi size.lo). phys.hi should contain the
+        // device's BDF as 0b00000000 bbbbbbbb dddddfff 00000000. The other cells
+        // should be zero.
+        let reg = [virtio_iommu_bdf << 8, 0_u32, 0_u32, 0_u32, 0_u32];
+        fdt.property_array_u32("reg", &reg)?;
+        fdt.property_u32("phandle", VIRTIO_IOMMU_PHANDLE)?;
+
+        fdt.end_node(virtio_iommu_node)?;
+    }
+
     fdt.end_node(pci_node)?;
 
     Ok(())
diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs
index fe83c6da0..16b6cec17 100644
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -142,6 +142,7 @@ pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::Bui
     device_info: &HashMap<(DeviceType, String), T, S>,
     initrd: &Option<super::InitramfsConfig>,
     pci_space_address: &(u64, u64),
+    virtio_iommu_bdf: Option<u32>,
     gic_device: &dyn GicDevice,
     numa_nodes: &NumaNodes,
 ) -> super::Result<()> {
@@ -155,6 +156,7 @@ pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::Bui
         initrd,
         pci_space_address,
         numa_nodes,
+        virtio_iommu_bdf,
     )
     .map_err(|_| Error::SetupFdt)?;
 
diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs
index 9539b23d8..c9e637f8c 100644
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -1077,6 +1077,14 @@ impl Vm {
 
         let pci_space = (pci_space_start.0, pci_space_size);
 
+        let virtio_iommu_bdf = self
+            .device_manager
+            .lock()
+            .unwrap()
+            .iommu_attached_devices()
+            .as_ref()
+            .map(|(v, _)| *v);
+
         #[cfg(feature = "acpi")]
         {
             let _ = crate::acpi::create_acpi_tables(
@@ -1104,6 +1112,7 @@ impl Vm {
             device_info,
             &initramfs_config,
             &pci_space,
+            virtio_iommu_bdf,
             &*gic_device,
             &self.numa_nodes,
         )
@@ -2584,6 +2593,7 @@ mod tests {
             &None,
             &(0x1_0000_0000, 0x1_0000),
             &BTreeMap::new(),
+            None,
         )
         .is_ok())
     }