tests: add ivshmem integration test case

Signed-off-by: Songqian Li <sionli@tencent.com>

scripts/run_integration_tests_aarch64.sh

@@ -258,4 +258,11 @@ if [ $RES -eq 0 ]; then
     RES=$?
 fi
 
+if [ $RES -eq 0 ]; then
+    cargo build --features ivshmem --all --release --target "$BUILD_TARGET"
+    export RUST_BACKTRACE=1
+    time cargo test "ivshmem::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]}
+    RES=$?
+fi
+
 exit $RES

scripts/run_integration_tests_x86_64.sh

@@ -206,4 +206,11 @@ if [ $RES -eq 0 ]; then
     RES=$?
 fi
 
+if [ $RES -eq 0 ]; then
+    cargo build --features ivshmem --all --release --target "$BUILD_TARGET"
+    export RUST_BACKTRACE=1
+    time cargo test $test_features "ivshmem::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]}
+    RES=$?
+fi
+
 exit $RES

test_infra/src/lib.rs

@@ -1776,3 +1776,27 @@ pub fn measure_virtio_net_latency(guest: &Guest, test_timeout: u32) -> Result<Ve
     let content = fs::read(log_file).map_err(Error::EthrLogFile)?;
     parse_ethr_latency_output(&content)
 }
+
+// parse the bar address from the output of `lspci -vv`
+
+pub fn extract_bar_address(output: &str, device_desc: &str, bar_index: usize) -> Option<String> {
+    let devices: Vec<&str> = output.split("\n\n").collect();
+
+    for device in devices {
+        if device.contains(device_desc) {
+            for line in device.lines() {
+                let line = line.trim();
+                let line_start_str = format!("Region {bar_index}: Memory at");
+                // for example: Region 2: Memory at 200000000 (64-bit, non-prefetchable) [size=1M]
+                if line.starts_with(line_start_str.as_str()) {
+                    let parts: Vec<&str> = line.split_whitespace().collect();
+                    if parts.len() >= 4 {
+                        let addr_str = parts[4];
+                        return Some(String::from(addr_str));
+                    }
+                }
+            }
+        }
+    }
+    None
+}

tests/integration.rs

@@ -11,7 +11,9 @@
 extern crate test_infra;
 
 use std::collections::HashMap;
-use std::io::{BufRead, Read, Seek, Write};
+use std::ffi::CStr;
+use std::fs::OpenOptions;
+use std::io::{BufRead, Read, Seek, SeekFrom, Write};
 use std::net::TcpListener;
 use std::os::unix::io::AsRawFd;
 use std::path::PathBuf;
@@ -2341,6 +2343,147 @@ fn make_guest_panic(guest: &Guest) {
     guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap();
 }
 
+// ivshmem test
+// This case validates that read data from host(host write data to ivshmem backend file,
+// guest read data from ivshmem pci bar2 memory)
+// and write data to host(guest write data to ivshmem pci bar2 memory, host read it from
+// ivshmem backend file).
+// It also checks the size of the shared memory region.
+fn _test_ivshmem(guest: &Guest, ivshmem_file_path: String, file_size: &str) {
+    let test_message_read = String::from("ivshmem device test data read");
+    // Modify backend file data before function test
+    let mut file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(ivshmem_file_path.as_str())
+        .unwrap();
+    file.seek(SeekFrom::Start(0)).unwrap();
+    file.write_all(test_message_read.as_bytes()).unwrap();
+    file.write_all(b"\0").unwrap();
+    file.flush().unwrap();
+
+    let output = fs::read_to_string(ivshmem_file_path.as_str()).unwrap();
+    let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap();
+    let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap();
+    let file_message = c_str.to_string_lossy().to_string();
+    // Check if the backend file data is correct
+    assert_eq!(test_message_read, file_message);
+
+    let device_id_line = String::from(
+        guest
+            .ssh_command("lspci -D | grep \"Inter-VM shared memory\"")
+            .unwrap()
+            .trim(),
+    );
+    // Check if ivshmem exists
+    assert!(!device_id_line.is_empty());
+    let device_id = device_id_line.split(" ").next().unwrap();
+    // Check shard memory size
+    assert_eq!(
+        guest
+            .ssh_command(
+                format!("lspci -vv -s {device_id} | grep -c \"Region 2.*size={file_size}\"")
+                    .as_str(),
+            )
+            .unwrap()
+            .trim()
+            .parse::<u32>()
+            .unwrap_or_default(),
+        1
+    );
+
+    // guest don't have gcc or g++, try to use python to test :(
+    // This python program try to mmap the ivshmem pci bar2 memory and read the data from it.
+    let ivshmem_test_read = format!(
+        r#"
+import os
+import mmap
+from ctypes import create_string_buffer, c_char, memmove
+
+if __name__ == "__main__":
+    device_path = f"/sys/bus/pci/devices/{device_id}/resource2"
+    fd = os.open(device_path, os.O_RDWR | os.O_SYNC)
+
+    PAGE_SIZE = os.sysconf('SC_PAGESIZE')
+
+    with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED,
+                      prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem:
+        c_buf = (c_char * PAGE_SIZE).from_buffer(shmem)
+        null_pos = c_buf.raw.find(b'\x00')
+        valid_data = c_buf.raw[:null_pos] if null_pos != -1 else c_buf.raw
+        print(valid_data.decode('utf-8', errors='replace'), end="")
+        shmem.flush()
+        del c_buf
+
+    os.close(fd)
+    "#
+    );
+    guest
+        .ssh_command(
+            format!(
+                r#"cat << EOF > test_read.py
+{ivshmem_test_read}
+EOF
+"#
+            )
+            .as_str(),
+        )
+        .unwrap();
+    let guest_message = guest.ssh_command("sudo python3 test_read.py").unwrap();
+
+    // Check the probe message in host and guest
+    assert_eq!(test_message_read, guest_message);
+
+    let test_message_write = "ivshmem device test data write";
+    // Then the program writes a test message to the memory and flush it.
+    let ivshmem_test_write = format!(
+        r#"
+import os
+import mmap
+from ctypes import create_string_buffer, c_char, memmove
+
+if __name__ == "__main__":
+    device_path = f"/sys/bus/pci/devices/{device_id}/resource2"
+    test_message = "{test_message_write}"
+    fd = os.open(device_path, os.O_RDWR | os.O_SYNC)
+
+    PAGE_SIZE = os.sysconf('SC_PAGESIZE')
+
+    with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED,
+                      prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem:
+        shmem.flush()
+        c_buf = (c_char * PAGE_SIZE).from_buffer(shmem)
+        encoded_msg = test_message.encode('utf-8').ljust(1000, b'\x00')
+        memmove(c_buf, encoded_msg, len(encoded_msg))
+        shmem.flush()
+        del c_buf
+
+    os.close(fd)
+    "#
+    );
+
+    guest
+        .ssh_command(
+            format!(
+                r#"cat << EOF > test_write.py
+{ivshmem_test_write}
+EOF
+"#
+            )
+            .as_str(),
+        )
+        .unwrap();
+
+    let _ = guest.ssh_command("sudo python3 test_write.py").unwrap();
+
+    let output = fs::read_to_string(ivshmem_file_path.as_str()).unwrap();
+    let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap();
+    let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap();
+    let file_message = c_str.to_string_lossy().to_string();
+    // Check to send data from guest to host
+    assert_eq!(test_message_write, file_message);
+}
+
 mod common_parallel {
     use std::fs::OpenOptions;
     use std::io::SeekFrom;
@@ -7275,6 +7418,226 @@ mod dbus_api {
     }
 }
 
+mod ivshmem {
+    use std::fs::remove_dir_all;
+    use std::process::Command;
+
+    use test_infra::{handle_child_output, kill_child, Guest, GuestCommand, UbuntuDiskConfig};
+
+    use crate::*;
+
+    #[test]
+    fn test_ivshmem() {
+        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(focal));
+        let api_socket = temp_api_path(&guest.tmp_dir);
+
+        let kernel_path = direct_kernel_boot_path();
+
+        let ivshmem_file_path = String::from(
+            guest
+                .tmp_dir
+                .as_path()
+                .join("ivshmem.data")
+                .to_str()
+                .unwrap(),
+        );
+        let file_size = "1M";
+
+        // Create a file to be used as the shared memory
+        Command::new("dd")
+            .args([
+                "if=/dev/zero",
+                format!("of={ivshmem_file_path}").as_str(),
+                format!("bs={file_size}").as_str(),
+                "count=1",
+            ])
+            .status()
+            .unwrap();
+
+        let mut child = GuestCommand::new(&guest)
+            .args(["--cpus", "boot=2"])
+            .args(["--memory", "size=512M"])
+            .args(["--kernel", kernel_path.to_str().unwrap()])
+            .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
+            .default_disks()
+            .default_net()
+            .args([
+                "--ivshmem",
+                format!("path={ivshmem_file_path},size={file_size}").as_str(),
+            ])
+            .args(["--api-socket", &api_socket])
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let r = std::panic::catch_unwind(|| {
+            guest.wait_vm_boot(None).unwrap();
+            _test_ivshmem(&guest, ivshmem_file_path, file_size);
+        });
+        kill_child(&mut child);
+        let output = child.wait_with_output().unwrap();
+
+        handle_child_output(r, &output);
+    }
+
+    #[test]
+    fn test_snapshot_restore_ivshmem() {
+        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(focal));
+        let kernel_path = direct_kernel_boot_path();
+
+        let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir));
+
+        let ivshmem_file_path = String::from(
+            guest
+                .tmp_dir
+                .as_path()
+                .join("ivshmem.data")
+                .to_str()
+                .unwrap(),
+        );
+        let file_size = "1M";
+
+        let device_params = {
+            let mut data = vec![];
+            // Create a file to be used as the shared memory
+            Command::new("dd")
+                .args([
+                    "if=/dev/zero",
+                    format!("of={ivshmem_file_path}").as_str(),
+                    format!("bs={file_size}").as_str(),
+                    "count=1",
+                ])
+                .status()
+                .unwrap();
+            data.push(String::from("--ivshmem"));
+            data.push(format!("path={ivshmem_file_path},size={file_size}"));
+            data
+        };
+
+        let socket = temp_vsock_path(&guest.tmp_dir);
+        let event_path = temp_event_monitor_path(&guest.tmp_dir);
+
+        let mut child = GuestCommand::new(&guest)
+            .args(["--api-socket", &api_socket_source])
+            .args(["--event-monitor", format!("path={event_path}").as_str()])
+            .args(["--cpus", "boot=2"])
+            .args(["--memory", "size=1G"])
+            .args(["--kernel", kernel_path.to_str().unwrap()])
+            .default_disks()
+            .default_net()
+            .args(["--vsock", format!("cid=3,socket={socket}").as_str()])
+            .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
+            .args(device_params)
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let console_text = String::from("On a branch floating down river a cricket, singing.");
+        // Create the snapshot directory
+        let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir);
+
+        let r = std::panic::catch_unwind(|| {
+            guest.wait_vm_boot(None).unwrap();
+
+            // Check the number of vCPUs
+            assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2);
+
+            common_sequential::snapshot_and_check_events(
+                &api_socket_source,
+                &snapshot_dir,
+                &event_path,
+            );
+        });
+
+        // Shutdown the source VM and check console output
+        kill_child(&mut child);
+        let output = child.wait_with_output().unwrap();
+        handle_child_output(r, &output);
+
+        // Remove the vsock socket file.
+        Command::new("rm")
+            .arg("-f")
+            .arg(socket.as_str())
+            .output()
+            .unwrap();
+
+        let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir));
+        let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir));
+
+        // Restore the VM from the snapshot
+        let mut child = GuestCommand::new(&guest)
+            .args(["--api-socket", &api_socket_restored])
+            .args([
+                "--event-monitor",
+                format!("path={event_path_restored}").as_str(),
+            ])
+            .args([
+                "--restore",
+                format!("source_url=file://{snapshot_dir}").as_str(),
+            ])
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        // Wait for the VM to be restored
+        thread::sleep(std::time::Duration::new(20, 0));
+
+        let latest_events = [&MetaEvent {
+            event: "restored".to_string(),
+            device_id: None,
+        }];
+        assert!(check_latest_events_exact(
+            &latest_events,
+            &event_path_restored
+        ));
+
+        // Remove the snapshot dir
+        let _ = remove_dir_all(snapshot_dir.as_str());
+
+        let r = std::panic::catch_unwind(|| {
+            // Resume the VM
+            assert!(remote_command(&api_socket_restored, "resume", None));
+            // There is no way that we can ensure the 'write()' to the
+            // event file is completed when the 'resume' request is
+            // returned successfully, because the 'write()' was done
+            // asynchronously from a different thread of Cloud
+            // Hypervisor (e.g. the event-monitor thread).
+            thread::sleep(std::time::Duration::new(1, 0));
+            let latest_events = [
+                &MetaEvent {
+                    event: "resuming".to_string(),
+                    device_id: None,
+                },
+                &MetaEvent {
+                    event: "resumed".to_string(),
+                    device_id: None,
+                },
+            ];
+            assert!(check_latest_events_exact(
+                &latest_events,
+                &event_path_restored
+            ));
+
+            // Check the number of vCPUs
+            assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2);
+            guest.check_devices_common(Some(&socket), Some(&console_text), None);
+            _test_ivshmem(&guest, ivshmem_file_path, file_size);
+        });
+        // Shutdown the target VM and check console output
+        kill_child(&mut child);
+        let output = child.wait_with_output().unwrap();
+        handle_child_output(r, &output);
+
+        let r = std::panic::catch_unwind(|| {
+            assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text));
+        });
+
+        handle_child_output(r, &output);
+    }
+}
+
 mod common_sequential {
     use std::fs::remove_dir_all;
 
@@ -7286,7 +7649,11 @@ mod common_sequential {
         test_memory_mergeable(true)
     }
 
-    fn snapshot_and_check_events(api_socket: &str, snapshot_dir: &str, event_path: &str) {
+    pub(crate) fn snapshot_and_check_events(
+        api_socket: &str,
+        snapshot_dir: &str,
+        event_path: &str,
+    ) {
         // Pause the VM
         assert!(remote_command(api_socket, "pause", None));
         let latest_events: [&MetaEvent; 2] = [
@@ -7833,7 +8200,7 @@ mod common_sequential {
         let device_params = {
             let mut data = vec![];
             if pvpanic {
-                data.push("--pvpanic");
+                data.push(String::from("--pvpanic"));
             }
             data
         };