From 90034fd6badc50764021dc5729fbfde2a784f073 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 24 May 2022 08:35:15 +0800 Subject: [PATCH] vmm: add GuestDebuggable trait It's useful to dump the guest, which named coredump so that crash tool can be used to analysize it when guest hung up. Let's add GuestDebuggable trait and Coredumpxxx error to support coredump firstly. Signed-off-by: Yi Wang Co-authored-by: Sebastien Boeuf --- vmm/Cargo.toml | 1 + vmm/src/coredump.rs | 43 ++++++++++++++++++ vmm/src/lib.rs | 2 + vmm/src/memory_manager.rs | 31 +++++++++++++ vmm/src/migration.rs | 14 ++++++ vmm/src/vm.rs | 92 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 183 insertions(+) create mode 100644 vmm/src/coredump.rs diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index 3b02d62f9..1f3b0b381 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -10,6 +10,7 @@ amx = [] cmos = ["devices/cmos"] fwdebug = ["devices/fwdebug"] gdb = ["kvm"] +guest_debug = ["kvm"] kvm = ["hypervisor/kvm", "vfio-ioctls/kvm", "vm-device/kvm", "pci/kvm"] mshv = ["hypervisor/mshv", "virtio-devices/mshv", "vfio-ioctls/mshv", "vm-device/mshv", "pci/mshv"] tdx = ["arch/tdx", "hypervisor/tdx"] diff --git a/vmm/src/coredump.rs b/vmm/src/coredump.rs new file mode 100644 index 000000000..f61bc6d11 --- /dev/null +++ b/vmm/src/coredump.rs @@ -0,0 +1,43 @@ +// Copyright © 2022 ZTE Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fs::File; + +#[derive(Clone)] +pub struct CoredumpMemoryRegion { + pub mem_offset_in_elf: u64, + pub mem_size: u64, +} + +#[derive(Clone)] +pub struct CoredumpMemoryRegions { + pub ram_maps: std::collections::BTreeMap, +} + +/// Platform information +#[derive(Default)] +pub struct DumpState { + pub elf_note_size: isize, + pub elf_phdr_num: u16, + pub elf_sh_info: u32, + pub mem_offset: u64, + pub mem_info: Option, + pub file: Option, +} + +#[derive(Debug)] +pub enum GuestDebuggableError { + Coredump(anyhow::Error), + CoredumpFile(std::io::Error), +} + +pub trait GuestDebuggable: vm_migration::Pausable { + fn coredump( + &mut self, + _destination_url: &str, + ) -> std::result::Result<(), GuestDebuggableError> { + Ok(()) + } +} diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 11d876849..a9830ac27 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -52,6 +52,8 @@ mod acpi; pub mod api; mod clone3; pub mod config; +#[cfg(feature = "guest_debug")] +mod coredump; pub mod cpu; pub mod device_manager; pub mod device_tree; diff --git a/vmm/src/memory_manager.rs b/vmm/src/memory_manager.rs index ae47f5746..02428aea5 100644 --- a/vmm/src/memory_manager.rs +++ b/vmm/src/memory_manager.rs @@ -5,6 +5,10 @@ #[cfg(target_arch = "x86_64")] use crate::config::SgxEpcConfig; use crate::config::{HotplugMethod, MemoryConfig, MemoryZoneConfig}; +#[cfg(feature = "guest_debug")] +use crate::coredump::{CoredumpMemoryRegion, CoredumpMemoryRegions}; +#[cfg(feature = "guest_debug")] +use crate::coredump::{DumpState, GuestDebuggable, GuestDebuggableError}; use crate::migration::url_to_path; use crate::MEMORY_MANAGER_SNAPSHOT_ID; use crate::{GuestMemoryMmap, GuestRegionMmap}; @@ -18,6 +22,8 @@ use devices::ioapic; #[cfg(target_arch = "x86_64")] use libc::{MAP_NORESERVE, MAP_POPULATE, MAP_SHARED, PROT_READ, PROT_WRITE}; use serde::{Deserialize, Serialize}; +#[cfg(feature = "guest_debug")] +use std::collections::BTreeMap; use std::collections::HashMap; use std::convert::TryInto; use std::ffi; @@ -1846,6 +1852,31 @@ impl MemoryManager { pub fn acpi_address(&self) -> Option { self.acpi_address } + + pub fn num_guest_ram_mappings(&self) -> u32 { + self.guest_ram_mappings.len() as u32 + } + + #[cfg(feature = "guest_debug")] + pub fn coredump_memory_regions(&self, mem_offset: u64) -> CoredumpMemoryRegions { + let mut mapping_sorted_by_gpa = self.guest_ram_mappings.clone(); + mapping_sorted_by_gpa.sort_by_key(|m| m.gpa); + + let mut mem_offset_in_elf = mem_offset; + let mut ram_maps = BTreeMap::new(); + for mapping in mapping_sorted_by_gpa.iter() { + ram_maps.insert( + mapping.gpa, + CoredumpMemoryRegion { + mem_offset_in_elf, + mem_size: mapping.size, + }, + ); + mem_offset_in_elf += mapping.size; + } + + CoredumpMemoryRegions { ram_maps } + } } struct MemoryNotify { diff --git a/vmm/src/migration.rs b/vmm/src/migration.rs index e0e0daa0c..63680aff1 100644 --- a/vmm/src/migration.rs +++ b/vmm/src/migration.rs @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 +#[cfg(feature = "guest_debug")] +use crate::coredump::GuestDebuggableError; use crate::{ config::VmConfig, vm::{VmSnapshot, VM_SNAPSHOT_ID}, @@ -32,6 +34,18 @@ pub fn url_to_path(url: &str) -> std::result::Result { Ok(path) } +#[cfg(feature = "guest_debug")] +pub fn url_to_file(url: &str) -> std::result::Result { + let file: PathBuf = url + .strip_prefix("file://") + .ok_or_else(|| { + GuestDebuggableError::Coredump(anyhow!("Could not extract file from URL: {}", url)) + }) + .map(|s| s.into())?; + + Ok(file) +} + pub fn recv_vm_config(source_url: &str) -> std::result::Result { let mut vm_config_path = url_to_path(source_url)?; diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 122ae5dc8..c1832e880 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -16,6 +16,10 @@ use crate::config::{ add_to_config, DeviceConfig, DiskConfig, FsConfig, HotplugMethod, NetConfig, PmemConfig, UserDeviceConfig, ValidationError, VdpaConfig, VmConfig, VsockConfig, }; +#[cfg(feature = "guest_debug")] +use crate::coredump::DumpState; +#[cfg(feature = "guest_debug")] +use crate::coredump::{GuestDebuggable, GuestDebuggableError}; use crate::cpu; use crate::device_manager::{Console, DeviceManager, DeviceManagerError, PtyPair}; use crate::device_tree::DeviceTree; @@ -24,6 +28,8 @@ use crate::gdb::{Debuggable, DebuggableError, GdbRequestPayload, GdbResponsePayl use crate::memory_manager::{ Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData, }; +#[cfg(feature = "guest_debug")] +use crate::migration::url_to_file; use crate::migration::{get_vm_snapshot, url_to_path, SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE}; use crate::seccomp_filters::{get_seccomp_filter, Thread}; use crate::GuestMemoryMmap; @@ -51,6 +57,8 @@ use devices::AcpiNotificationFlags; use gdbstub_arch::x86::reg::X86_64CoreRegs; use hypervisor::{HypervisorVmError, VmOps}; use linux_loader::cmdline::Cmdline; +#[cfg(feature = "guest_debug")] +use linux_loader::elf; #[cfg(target_arch = "x86_64")] use linux_loader::loader::elf::PvhBootCapability::PvhEntryPresent; #[cfg(target_arch = "aarch64")] @@ -72,6 +80,8 @@ use std::io::{self, Read, Write}; use std::io::{Seek, SeekFrom}; #[cfg(feature = "tdx")] use std::mem; +#[cfg(feature = "guest_debug")] +use std::mem::size_of; use std::num::Wrapping; use std::ops::Deref; use std::os::unix::net::UnixStream; @@ -294,6 +304,10 @@ pub enum Error { #[cfg(target_arch = "x86_64")] #[error("Error joining kernel loading thread")] KernelLoadThreadJoin(std::boxed::Box), + + #[cfg(feature = "guest_debug")] + #[error("Error coredumping VM: {0:?}")] + Coredump(GuestDebuggableError), } pub type Result = result::Result; @@ -2527,6 +2541,54 @@ impl Vm { } Ok(GdbResponsePayload::CommandComplete) } + + #[cfg(feature = "guest_debug")] + fn get_dump_state( + &mut self, + destination_url: &str, + ) -> std::result::Result { + let nr_cpus = self.config.lock().unwrap().cpus.boot_vcpus as u32; + let elf_note_size = self.get_note_size(NoteDescType::ElfAndVmmDesc, nr_cpus) as isize; + let mut elf_phdr_num = 1 as u16; + let elf_sh_info = 0; + let coredump_file_path = url_to_file(destination_url)?; + let mapping_num = self.memory_manager.lock().unwrap().num_guest_ram_mappings(); + + if mapping_num < UINT16_MAX - 2 { + elf_phdr_num += mapping_num as u16; + } else { + panic!("mapping num beyond 65535 not supported"); + } + let coredump_file = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(coredump_file_path) + .map_err(|e| GuestDebuggableError::Coredump(e.into()))?; + + let mem_offset = self.coredump_get_mem_offset(elf_phdr_num, elf_note_size); + let mem_data = self + .memory_manager + .lock() + .unwrap() + .coredump_memory_regions(mem_offset); + + Ok(DumpState { + elf_note_size, + elf_phdr_num, + elf_sh_info, + mem_offset, + mem_info: Some(mem_data), + file: Some(coredump_file), + }) + } + + #[cfg(feature = "guest_debug")] + fn coredump_get_mem_offset(&self, phdr_num: u16, note_size: isize) -> u64 { + size_of::() as u64 + + note_size as u64 + + size_of::() as u64 * phdr_num as u64 + } } impl Pausable for Vm { @@ -2955,6 +3017,36 @@ impl Debuggable for Vm { } } +#[cfg(feature = "guest_debug")] +pub const UINT16_MAX: u32 = 65535; + +#[cfg(feature = "guest_debug")] +impl GuestDebuggable for Vm { + fn coredump(&mut self, destination_url: &str) -> std::result::Result<(), GuestDebuggableError> { + event!("vm", "coredumping"); + + #[cfg(feature = "tdx")] + { + if self.config.lock().unwrap().tdx.is_some() { + return Err(GuestDebuggableError::Coredump(anyhow!( + "Coredump not possible with TDX VM" + ))); + } + } + + let current_state = self.get_state().unwrap(); + if current_state != VmState::Paused { + return Err(GuestDebuggableError::Coredump(anyhow!( + "Trying to coredump while VM is running" + ))); + } + + let coredump_state = self.get_dump_state(destination_url)?; + + Ok(()) + } +} + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] #[cfg(test)] mod tests {