From 28e12e9f3ac3f733ff3ca1f5f3d04273912fd9c7 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 16 Oct 2020 18:12:46 +0200 Subject: [PATCH] vmm, hypervisor: Fix snapshot/restore for Windows guest The snasphot/restore feature is not working because some CPU states are not properly saved, which means they can't be restored later on. First thing, we ensure the CPUID is stored so that it can be properly restored later. The code is simplified and pushed down to the hypervisor crate. Second thing, we identify for each vCPU if the Hyper-V SynIC device is emulated or not. In case it is, that means some specific MSRs will be set by the guest. These MSRs must be saved in order to properly restore the VM. Signed-off-by: Sebastien Boeuf --- hypervisor/src/kvm/mod.rs | 37 +++++++++++++++++++++++++++++++- hypervisor/src/kvm/x86_64/mod.rs | 1 + vmm/src/cpu.rs | 17 ++------------- vmm/src/seccomp_filters.rs | 2 ++ 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs index b03e39d8e..6c1fd0631 100644 --- a/hypervisor/src/kvm/mod.rs +++ b/hypervisor/src/kvm/mod.rs @@ -24,6 +24,8 @@ use kvm_ioctls::{NoDatamatch, VcpuFd, VmFd}; use serde_derive::{Deserialize, Serialize}; use std::os::unix::io::{AsRawFd, RawFd}; use std::result; +#[cfg(target_arch = "x86_64")] +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; #[cfg(target_arch = "x86_64")] use vm_memory::Address; @@ -47,7 +49,9 @@ pub use x86_64::{ }; #[cfg(target_arch = "x86_64")] -use kvm_bindings::{kvm_enable_cap, MsrList, KVM_CAP_HYPERV_SYNIC, KVM_CAP_SPLIT_IRQCHIP}; +use kvm_bindings::{ + kvm_enable_cap, kvm_msr_entry, MsrList, KVM_CAP_HYPERV_SYNIC, KVM_CAP_SPLIT_IRQCHIP, +}; #[cfg(target_arch = "x86_64")] use crate::arch::x86::NUM_IOAPIC_PINS; @@ -182,6 +186,8 @@ impl vm::Vm for KvmVm { #[cfg(target_arch = "x86_64")] msrs: self.msrs.clone(), vmmops: self.vmmops.clone(), + #[cfg(target_arch = "x86_64")] + hyperv_synic: AtomicBool::new(false), }; Ok(Arc::new(vcpu)) } @@ -504,6 +510,8 @@ pub struct KvmVcpu { #[cfg(target_arch = "x86_64")] msrs: MsrEntries, vmmops: ArcSwapOption>, + #[cfg(target_arch = "x86_64")] + hyperv_synic: AtomicBool, } /// Implementation of Vcpu trait for KVM /// Example: @@ -584,6 +592,10 @@ impl cpu::Vcpu for KvmVcpu { /// X86 specific call to enable HyperV SynIC /// fn enable_hyperv_synic(&self) -> cpu::Result<()> { + // Update the information about Hyper-V SynIC being enabled and + // emulated as it will influence later which MSRs should be saved. + self.hyperv_synic.store(true, Ordering::SeqCst); + let mut cap: kvm_enable_cap = Default::default(); cap.cap = KVM_CAP_HYPERV_SYNIC; self.fd @@ -1113,6 +1125,7 @@ impl cpu::Vcpu for KvmVcpu { /// let state = vcpu.state().unwrap(); /// ``` fn state(&self) -> cpu::Result { + let cpuid = self.get_cpuid2(kvm_bindings::KVM_MAX_CPUID_ENTRIES)?; let mp_state = self.get_mp_state()?; let regs = self.get_regs()?; let sregs = self.get_sregs()?; @@ -1127,6 +1140,26 @@ impl cpu::Vcpu for KvmVcpu { // by chunks. This is the only way to make sure we try to get as many // MSRs as possible, even if some MSRs are not supported. let mut msr_entries = self.msrs.clone(); + + // Save extra MSRs if the Hyper-V synthetic interrupt controller is + // emulated. + if self.hyperv_synic.load(Ordering::SeqCst) { + let hyperv_synic_msrs = vec![ + 0x40000020, 0x40000021, 0x40000080, 0x40000081, 0x40000082, 0x40000083, 0x40000084, + 0x40000090, 0x40000091, 0x40000092, 0x40000093, 0x40000094, 0x40000095, 0x40000096, + 0x40000097, 0x40000098, 0x40000099, 0x4000009a, 0x4000009b, 0x4000009c, 0x4000009d, + 0x4000009f, 0x400000b0, 0x400000b1, 0x400000b2, 0x400000b3, 0x400000b4, 0x400000b5, + 0x400000b6, 0x400000b7, + ]; + for index in hyperv_synic_msrs { + let msr = kvm_msr_entry { + index, + ..Default::default() + }; + msr_entries.push(msr).unwrap(); + } + } + let expected_num_msrs = msr_entries.as_fam_struct_ref().nmsrs as usize; let num_msrs = self.get_msrs(&mut msr_entries)?; let msrs = if num_msrs != expected_num_msrs { @@ -1172,6 +1205,7 @@ impl cpu::Vcpu for KvmVcpu { let vcpu_events = self.get_vcpu_events()?; Ok(CpuState { + cpuid, msrs, vcpu_events, regs, @@ -1238,6 +1272,7 @@ impl cpu::Vcpu for KvmVcpu { /// vcpu.set_state(&state).unwrap(); /// ``` fn set_state(&self, state: &CpuState) -> cpu::Result<()> { + self.set_cpuid2(&state.cpuid)?; self.set_mp_state(state.mp_state)?; self.set_regs(&state.regs)?; self.set_sregs(&state.sregs)?; diff --git a/hypervisor/src/kvm/x86_64/mod.rs b/hypervisor/src/kvm/x86_64/mod.rs index f39a7c581..3f3343033 100644 --- a/hypervisor/src/kvm/x86_64/mod.rs +++ b/hypervisor/src/kvm/x86_64/mod.rs @@ -85,6 +85,7 @@ pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> { } #[derive(Clone, Serialize, Deserialize)] pub struct VcpuKvmState { + pub cpuid: CpuId, pub msrs: MsrEntries, pub vcpu_events: VcpuEvents, pub regs: StandardRegisters, diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs index 633de16db..df0f9257e 100644 --- a/vmm/src/cpu.rs +++ b/vmm/src/cpu.rs @@ -26,9 +26,9 @@ use anyhow::anyhow; use arch::layout; #[cfg(target_arch = "x86_64")] use arch::x86_64::SgxEpcSection; -use arch::EntryPoint; #[cfg(target_arch = "x86_64")] -use arch::{CpuidPatch, CpuidReg}; +use arch::CpuidPatch; +use arch::EntryPoint; use devices::interrupt_controller::InterruptController; #[cfg(target_arch = "aarch64")] use hypervisor::kvm::kvm_bindings; @@ -688,19 +688,6 @@ impl CpuManager { let vcpu = Vcpu::new(cpu_id, &self.vm, interrupt_controller)?; if let Some(snapshot) = snapshot { - #[cfg(target_arch = "x86_64")] - { - let mut cpuid = self.cpuid.clone(); - CpuidPatch::set_cpuid_reg(&mut cpuid, 0xb, None, CpuidReg::EDX, u32::from(cpu_id)); - CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1f, None, CpuidReg::EDX, u32::from(cpu_id)); - - vcpu.lock() - .unwrap() - .vcpu - .set_cpuid2(&cpuid) - .map_err(|e| Error::SetSupportedCpusFailed(e.into()))?; - } - // AArch64 vCPUs should be initialized after created. #[cfg(target_arch = "aarch64")] vcpu.lock().unwrap().init(&self.vm)?; diff --git a/vmm/src/seccomp_filters.rs b/vmm/src/seccomp_filters.rs index 3a208f564..4a178ac9d 100644 --- a/vmm/src/seccomp_filters.rs +++ b/vmm/src/seccomp_filters.rs @@ -183,6 +183,7 @@ fn create_vmm_ioctl_seccomp_rule_common() -> Result, Error> { fn create_vmm_ioctl_seccomp_rule() -> Result, Error> { const KVM_CREATE_PIT2: u64 = 0x4040_ae77; const KVM_GET_CLOCK: u64 = 0x8030_ae7c; + const KVM_GET_CPUID2: u64 = 0xc008_ae91; const KVM_GET_FPU: u64 = 0x81a0_ae8c; const KVM_GET_LAPIC: u64 = 0x8400_ae8e; const KVM_GET_MSR_INDEX_LIST: u64 = 0xc004_ae02; @@ -205,6 +206,7 @@ fn create_vmm_ioctl_seccomp_rule() -> Result, Error> { let mut arch_rules = or![ and![Cond::new(1, ArgLen::DWORD, Eq, KVM_CREATE_PIT2)?], and![Cond::new(1, ArgLen::DWORD, Eq, KVM_GET_CLOCK,)?], + and![Cond::new(1, ArgLen::DWORD, Eq, KVM_GET_CPUID2,)?], and![Cond::new(1, ArgLen::DWORD, Eq, KVM_GET_FPU)?], and![Cond::new(1, ArgLen::DWORD, Eq, KVM_GET_LAPIC)?], and![Cond::new(1, ArgLen::DWORD, Eq, KVM_GET_MSR_INDEX_LIST)?],