diff --git a/scripts/run_integration_tests_aarch64.sh b/scripts/run_integration_tests_aarch64.sh index 262faff9a..758c69c6b 100755 --- a/scripts/run_integration_tests_aarch64.sh +++ b/scripts/run_integration_tests_aarch64.sh @@ -258,4 +258,11 @@ if [ $RES -eq 0 ]; then RES=$? fi +if [ $RES -eq 0 ]; then + cargo build --features ivshmem --all --release --target "$BUILD_TARGET" + export RUST_BACKTRACE=1 + time cargo test "ivshmem::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]} + RES=$? +fi + exit $RES diff --git a/scripts/run_integration_tests_x86_64.sh b/scripts/run_integration_tests_x86_64.sh index 4f4491aa7..3f28e23cd 100755 --- a/scripts/run_integration_tests_x86_64.sh +++ b/scripts/run_integration_tests_x86_64.sh @@ -206,4 +206,11 @@ if [ $RES -eq 0 ]; then RES=$? fi +if [ $RES -eq 0 ]; then + cargo build --features ivshmem --all --release --target "$BUILD_TARGET" + export RUST_BACKTRACE=1 + time cargo test $test_features "ivshmem::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]} + RES=$? +fi + exit $RES diff --git a/test_infra/src/lib.rs b/test_infra/src/lib.rs index df47de835..6875aa5b2 100644 --- a/test_infra/src/lib.rs +++ b/test_infra/src/lib.rs @@ -1776,3 +1776,27 @@ pub fn measure_virtio_net_latency(guest: &Guest, test_timeout: u32) -> Result Option { + let devices: Vec<&str> = output.split("\n\n").collect(); + + for device in devices { + if device.contains(device_desc) { + for line in device.lines() { + let line = line.trim(); + let line_start_str = format!("Region {bar_index}: Memory at"); + // for example: Region 2: Memory at 200000000 (64-bit, non-prefetchable) [size=1M] + if line.starts_with(line_start_str.as_str()) { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 4 { + let addr_str = parts[4]; + return Some(String::from(addr_str)); + } + } + } + } + } + None +} diff --git a/tests/integration.rs b/tests/integration.rs index dc19b7aee..d604a5f7c 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -11,7 +11,9 @@ extern crate test_infra; use std::collections::HashMap; -use std::io::{BufRead, Read, Seek, Write}; +use std::ffi::CStr; +use std::fs::OpenOptions; +use std::io::{BufRead, Read, Seek, SeekFrom, Write}; use std::net::TcpListener; use std::os::unix::io::AsRawFd; use std::path::PathBuf; @@ -2341,6 +2343,147 @@ fn make_guest_panic(guest: &Guest) { guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); } +// ivshmem test +// This case validates that read data from host(host write data to ivshmem backend file, +// guest read data from ivshmem pci bar2 memory) +// and write data to host(guest write data to ivshmem pci bar2 memory, host read it from +// ivshmem backend file). +// It also checks the size of the shared memory region. +fn _test_ivshmem(guest: &Guest, ivshmem_file_path: String, file_size: &str) { + let test_message_read = String::from("ivshmem device test data read"); + // Modify backend file data before function test + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(ivshmem_file_path.as_str()) + .unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + file.write_all(test_message_read.as_bytes()).unwrap(); + file.write_all(b"\0").unwrap(); + file.flush().unwrap(); + + let output = fs::read_to_string(ivshmem_file_path.as_str()).unwrap(); + let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap(); + let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap(); + let file_message = c_str.to_string_lossy().to_string(); + // Check if the backend file data is correct + assert_eq!(test_message_read, file_message); + + let device_id_line = String::from( + guest + .ssh_command("lspci -D | grep \"Inter-VM shared memory\"") + .unwrap() + .trim(), + ); + // Check if ivshmem exists + assert!(!device_id_line.is_empty()); + let device_id = device_id_line.split(" ").next().unwrap(); + // Check shard memory size + assert_eq!( + guest + .ssh_command( + format!("lspci -vv -s {device_id} | grep -c \"Region 2.*size={file_size}\"") + .as_str(), + ) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 + ); + + // guest don't have gcc or g++, try to use python to test :( + // This python program try to mmap the ivshmem pci bar2 memory and read the data from it. + let ivshmem_test_read = format!( + r#" +import os +import mmap +from ctypes import create_string_buffer, c_char, memmove + +if __name__ == "__main__": + device_path = f"/sys/bus/pci/devices/{device_id}/resource2" + fd = os.open(device_path, os.O_RDWR | os.O_SYNC) + + PAGE_SIZE = os.sysconf('SC_PAGESIZE') + + with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem: + c_buf = (c_char * PAGE_SIZE).from_buffer(shmem) + null_pos = c_buf.raw.find(b'\x00') + valid_data = c_buf.raw[:null_pos] if null_pos != -1 else c_buf.raw + print(valid_data.decode('utf-8', errors='replace'), end="") + shmem.flush() + del c_buf + + os.close(fd) + "# + ); + guest + .ssh_command( + format!( + r#"cat << EOF > test_read.py +{ivshmem_test_read} +EOF +"# + ) + .as_str(), + ) + .unwrap(); + let guest_message = guest.ssh_command("sudo python3 test_read.py").unwrap(); + + // Check the probe message in host and guest + assert_eq!(test_message_read, guest_message); + + let test_message_write = "ivshmem device test data write"; + // Then the program writes a test message to the memory and flush it. + let ivshmem_test_write = format!( + r#" +import os +import mmap +from ctypes import create_string_buffer, c_char, memmove + +if __name__ == "__main__": + device_path = f"/sys/bus/pci/devices/{device_id}/resource2" + test_message = "{test_message_write}" + fd = os.open(device_path, os.O_RDWR | os.O_SYNC) + + PAGE_SIZE = os.sysconf('SC_PAGESIZE') + + with mmap.mmap(fd, PAGE_SIZE, flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ | mmap.PROT_WRITE, offset=0) as shmem: + shmem.flush() + c_buf = (c_char * PAGE_SIZE).from_buffer(shmem) + encoded_msg = test_message.encode('utf-8').ljust(1000, b'\x00') + memmove(c_buf, encoded_msg, len(encoded_msg)) + shmem.flush() + del c_buf + + os.close(fd) + "# + ); + + guest + .ssh_command( + format!( + r#"cat << EOF > test_write.py +{ivshmem_test_write} +EOF +"# + ) + .as_str(), + ) + .unwrap(); + + let _ = guest.ssh_command("sudo python3 test_write.py").unwrap(); + + let output = fs::read_to_string(ivshmem_file_path.as_str()).unwrap(); + let nul_pos = output.as_bytes().iter().position(|&b| b == 0).unwrap(); + let c_str = CStr::from_bytes_until_nul(&output.as_bytes()[..=nul_pos]).unwrap(); + let file_message = c_str.to_string_lossy().to_string(); + // Check to send data from guest to host + assert_eq!(test_message_write, file_message); +} + mod common_parallel { use std::fs::OpenOptions; use std::io::SeekFrom; @@ -7275,6 +7418,226 @@ mod dbus_api { } } +mod ivshmem { + use std::fs::remove_dir_all; + use std::process::Command; + + use test_infra::{handle_child_output, kill_child, Guest, GuestCommand, UbuntuDiskConfig}; + + use crate::*; + + #[test] + fn test_ivshmem() { + let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(focal)); + let api_socket = temp_api_path(&guest.tmp_dir); + + let kernel_path = direct_kernel_boot_path(); + + let ivshmem_file_path = String::from( + guest + .tmp_dir + .as_path() + .join("ivshmem.data") + .to_str() + .unwrap(), + ); + let file_size = "1M"; + + // Create a file to be used as the shared memory + Command::new("dd") + .args([ + "if=/dev/zero", + format!("of={ivshmem_file_path}").as_str(), + format!("bs={file_size}").as_str(), + "count=1", + ]) + .status() + .unwrap(); + + let mut child = GuestCommand::new(&guest) + .args(["--cpus", "boot=2"]) + .args(["--memory", "size=512M"]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .default_net() + .args([ + "--ivshmem", + format!("path={ivshmem_file_path},size={file_size}").as_str(), + ]) + .args(["--api-socket", &api_socket]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot(None).unwrap(); + _test_ivshmem(&guest, ivshmem_file_path, file_size); + }); + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); + } + + #[test] + fn test_snapshot_restore_ivshmem() { + let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(focal)); + let kernel_path = direct_kernel_boot_path(); + + let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); + + let ivshmem_file_path = String::from( + guest + .tmp_dir + .as_path() + .join("ivshmem.data") + .to_str() + .unwrap(), + ); + let file_size = "1M"; + + let device_params = { + let mut data = vec![]; + // Create a file to be used as the shared memory + Command::new("dd") + .args([ + "if=/dev/zero", + format!("of={ivshmem_file_path}").as_str(), + format!("bs={file_size}").as_str(), + "count=1", + ]) + .status() + .unwrap(); + data.push(String::from("--ivshmem")); + data.push(format!("path={ivshmem_file_path},size={file_size}")); + data + }; + + let socket = temp_vsock_path(&guest.tmp_dir); + let event_path = temp_event_monitor_path(&guest.tmp_dir); + + let mut child = GuestCommand::new(&guest) + .args(["--api-socket", &api_socket_source]) + .args(["--event-monitor", format!("path={event_path}").as_str()]) + .args(["--cpus", "boot=2"]) + .args(["--memory", "size=1G"]) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .default_disks() + .default_net() + .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .args(device_params) + .capture_output() + .spawn() + .unwrap(); + + let console_text = String::from("On a branch floating down river a cricket, singing."); + // Create the snapshot directory + let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot(None).unwrap(); + + // Check the number of vCPUs + assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); + + common_sequential::snapshot_and_check_events( + &api_socket_source, + &snapshot_dir, + &event_path, + ); + }); + + // Shutdown the source VM and check console output + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + handle_child_output(r, &output); + + // Remove the vsock socket file. + Command::new("rm") + .arg("-f") + .arg(socket.as_str()) + .output() + .unwrap(); + + let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); + let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); + + // Restore the VM from the snapshot + let mut child = GuestCommand::new(&guest) + .args(["--api-socket", &api_socket_restored]) + .args([ + "--event-monitor", + format!("path={event_path_restored}").as_str(), + ]) + .args([ + "--restore", + format!("source_url=file://{snapshot_dir}").as_str(), + ]) + .capture_output() + .spawn() + .unwrap(); + + // Wait for the VM to be restored + thread::sleep(std::time::Duration::new(20, 0)); + + let latest_events = [&MetaEvent { + event: "restored".to_string(), + device_id: None, + }]; + assert!(check_latest_events_exact( + &latest_events, + &event_path_restored + )); + + // Remove the snapshot dir + let _ = remove_dir_all(snapshot_dir.as_str()); + + let r = std::panic::catch_unwind(|| { + // Resume the VM + assert!(remote_command(&api_socket_restored, "resume", None)); + // There is no way that we can ensure the 'write()' to the + // event file is completed when the 'resume' request is + // returned successfully, because the 'write()' was done + // asynchronously from a different thread of Cloud + // Hypervisor (e.g. the event-monitor thread). + thread::sleep(std::time::Duration::new(1, 0)); + let latest_events = [ + &MetaEvent { + event: "resuming".to_string(), + device_id: None, + }, + &MetaEvent { + event: "resumed".to_string(), + device_id: None, + }, + ]; + assert!(check_latest_events_exact( + &latest_events, + &event_path_restored + )); + + // Check the number of vCPUs + assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); + guest.check_devices_common(Some(&socket), Some(&console_text), None); + _test_ivshmem(&guest, ivshmem_file_path, file_size); + }); + // Shutdown the target VM and check console output + kill_child(&mut child); + let output = child.wait_with_output().unwrap(); + handle_child_output(r, &output); + + let r = std::panic::catch_unwind(|| { + assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); + }); + + handle_child_output(r, &output); + } +} + mod common_sequential { use std::fs::remove_dir_all; @@ -7286,7 +7649,11 @@ mod common_sequential { test_memory_mergeable(true) } - fn snapshot_and_check_events(api_socket: &str, snapshot_dir: &str, event_path: &str) { + pub(crate) fn snapshot_and_check_events( + api_socket: &str, + snapshot_dir: &str, + event_path: &str, + ) { // Pause the VM assert!(remote_command(api_socket, "pause", None)); let latest_events: [&MetaEvent; 2] = [ @@ -7833,7 +8200,7 @@ mod common_sequential { let device_params = { let mut data = vec![]; if pvpanic { - data.push("--pvpanic"); + data.push(String::from("--pvpanic")); } data };