From 4487c8376bd8e636f31f8b238da87a0f37faf280 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 30 Nov 2022 11:15:33 +0100 Subject: [PATCH] vmm: Move CpuManager and Vcpu to the new restore design Every Vcpu is now created with the right state if there's an available snapshot associated with it. This simplifies the restore logic. Signed-off-by: Sebastien Boeuf --- vmm/src/cpu.rs | 110 +++++++++++++++++++++++-------------------------- vmm/src/vm.rs | 15 +------ 2 files changed, 53 insertions(+), 72 deletions(-) diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs index cd0489f5f..68ddf19b3 100644 --- a/vmm/src/cpu.rs +++ b/vmm/src/cpu.rs @@ -76,8 +76,8 @@ use vm_memory::ByteValued; use vm_memory::{Bytes, GuestAddressSpace}; use vm_memory::{GuestAddress, GuestMemoryAtomic}; use vm_migration::{ - Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable, - Transportable, + snapshot_from_id, Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, + Snapshottable, Transportable, }; use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::{register_signal_handler, SIGRTMIN}; @@ -407,6 +407,8 @@ impl Snapshottable for Vcpu { .state() .map_err(|e| MigratableError::Pause(anyhow!("Could not get vCPU state {:?}", e)))?; + // TODO: The special format of the CPU id can be removed once ready to + // break live upgrade. let mut vcpu_snapshot = Snapshot::new(&format!("{:03}", self.id)); vcpu_snapshot.add_data_section(SnapshotDataSection::new_from_state( VCPU_SNAPSHOT_ID, @@ -417,18 +419,6 @@ impl Snapshottable for Vcpu { Ok(vcpu_snapshot) } - - fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> { - let saved_state: CpuState = snapshot.to_state(VCPU_SNAPSHOT_ID)?; - - self.vcpu - .set_state(&saved_state) - .map_err(|e| MigratableError::Pause(anyhow!("Could not set the vCPU state {:?}", e)))?; - - self.saved_state = Some(saved_state); - - Ok(()) - } } pub struct CpuManager { @@ -720,14 +710,27 @@ impl CpuManager { }))) } - fn create_vcpu(&mut self, cpu_id: u8) -> Result>> { + fn create_vcpu(&mut self, cpu_id: u8, snapshot: Option) -> Result>> { info!("Creating vCPU: cpu_id = {}", cpu_id); - let vcpu = Arc::new(Mutex::new(Vcpu::new( - cpu_id, - &self.vm, - Some(self.vm_ops.clone()), - )?)); + let mut vcpu = Vcpu::new(cpu_id, &self.vm, Some(self.vm_ops.clone()))?; + + if let Some(snapshot) = snapshot { + // AArch64 vCPUs should be initialized after created. + #[cfg(target_arch = "aarch64")] + vcpu.init(&self.vm)?; + + let state: CpuState = snapshot.to_state(VCPU_SNAPSHOT_ID).map_err(|e| { + Error::VcpuCreate(anyhow!("Could not get vCPU state from snapshot {:?}", e)) + })?; + vcpu.vcpu + .set_state(&state) + .map_err(|e| Error::VcpuCreate(anyhow!("Could not set the vCPU state {:?}", e)))?; + + vcpu.saved_state = Some(state); + } + + let vcpu = Arc::new(Mutex::new(vcpu)); // Adding vCPU to the CpuManager's vCPU list. self.vcpus.push(vcpu.clone()); @@ -739,36 +742,31 @@ impl CpuManager { &self, vcpu: Arc>, entry_point: Option, - snapshot: Option, ) -> Result<()> { let mut vcpu = vcpu.lock().unwrap(); - if let Some(snapshot) = snapshot { - // AArch64 vCPUs should be initialized after created. - #[cfg(target_arch = "aarch64")] - vcpu.init(&self.vm)?; + #[cfg(target_arch = "x86_64")] + vcpu.configure( + entry_point, + &self.vm_memory, + self.cpuid.clone(), + self.config.kvm_hyperv, + ) + .expect("Failed to configure vCPU"); - vcpu.restore(snapshot).expect("Failed to restore vCPU"); - } else { - #[cfg(target_arch = "x86_64")] - vcpu.configure( - entry_point, - &self.vm_memory, - self.cpuid.clone(), - self.config.kvm_hyperv, - ) + #[cfg(target_arch = "aarch64")] + vcpu.configure(&self.vm, entry_point) .expect("Failed to configure vCPU"); - #[cfg(target_arch = "aarch64")] - vcpu.configure(&self.vm, entry_point) - .expect("Failed to configure vCPU"); - } - Ok(()) } /// Only create new vCPUs if there aren't any inactive ones to reuse - fn create_vcpus(&mut self, desired_vcpus: u8) -> Result>>> { + fn create_vcpus( + &mut self, + desired_vcpus: u8, + snapshot: Option, + ) -> Result>>> { let mut vcpus: Vec>> = vec![]; info!( "Request to create new vCPUs: desired = {}, max = {}, allocated = {}, present = {}", @@ -784,7 +782,12 @@ impl CpuManager { // Only create vCPUs in excess of all the allocated vCPUs. for cpu_id in self.vcpus.len() as u8..desired_vcpus { - vcpus.push(self.create_vcpu(cpu_id)?); + vcpus.push(self.create_vcpu( + cpu_id, + // TODO: The special format of the CPU id can be removed once + // ready to break live upgrade. + snapshot_from_id(snapshot.as_ref(), &format!("{:03}", cpu_id)), + )?); } Ok(vcpus) @@ -1116,10 +1119,13 @@ impl CpuManager { Ok(()) } - pub fn create_boot_vcpus(&mut self) -> Result>>> { + pub fn create_boot_vcpus( + &mut self, + snapshot: Option, + ) -> Result>>> { trace_scoped!("create_boot_vcpus"); - self.create_vcpus(self.boot_vcpus()) + self.create_vcpus(self.boot_vcpus(), snapshot) } // Starts all the vCPUs that the VM is booting with. Blocks until all vCPUs are running. @@ -1147,9 +1153,9 @@ impl CpuManager { match desired_vcpus.cmp(&self.present_vcpus()) { cmp::Ordering::Greater => { - let vcpus = self.create_vcpus(desired_vcpus)?; + let vcpus = self.create_vcpus(desired_vcpus, None)?; for vcpu in vcpus { - self.configure_vcpu(vcpu, None, None)? + self.configure_vcpu(vcpu, None)? } self.activate_vcpus(desired_vcpus, true, None)?; Ok(true) @@ -2079,20 +2085,6 @@ impl Snapshottable for CpuManager { Ok(cpu_manager_snapshot) } - - fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> { - for (cpu_id, snapshot) in snapshot.snapshots.iter() { - let cpu_id = cpu_id.parse::().unwrap(); - info!("Restoring VCPU {}", cpu_id); - let vcpu = self.vcpus[cpu_id].clone(); - self.configure_vcpu(vcpu, None, Some(*snapshot.clone())) - .map_err(|e| { - MigratableError::Restore(anyhow!("Could not configure vCPU {:?}", e)) - })? - } - - Ok(()) - } } impl Transportable for CpuManager {} diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index c40802be6..e9be2725a 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -541,7 +541,7 @@ impl Vm { cpu_manager .lock() .unwrap() - .create_boot_vcpus() + .create_boot_vcpus(snapshot_from_id(snapshot, CPU_MANAGER_SNAPSHOT_ID)) .map_err(Error::CpuManager)?; #[cfg(feature = "tdx")] @@ -2083,7 +2083,7 @@ impl Vm { self.cpu_manager .lock() .unwrap() - .configure_vcpu(vcpu, entry_point, None) + .configure_vcpu(vcpu, entry_point) .map_err(Error::CpuManager)?; } @@ -2636,17 +2636,6 @@ impl Snapshottable for Vm { ))); } - if let Some(cpu_manager_snapshot) = snapshot.snapshots.get(CPU_MANAGER_SNAPSHOT_ID) { - self.cpu_manager - .lock() - .unwrap() - .restore(*cpu_manager_snapshot.clone())?; - } else { - return Err(MigratableError::Restore(anyhow!( - "Missing CPU manager snapshot" - ))); - } - #[cfg(target_arch = "aarch64")] self.restore_vgic_and_enable_interrupt(&snapshot)?;