Implement hot-unplug on the Guest side

Guest side hot-unplug implementation. This means that the API call can
be made, and the vCPUs are successfully removed from the guest, but the
backing vCPU threads are not removed from the host VMM.

As a result of this, hot-plugs that occur after hot-unplugs do not work
correctly right now. Once the total thread count in the VMM exceeds 32,
there is no effect of hot-plugging. To complete this, a refactor is
needed of the VMM, such that the CpuContainer can somehow remove the
threads when the guest kernel calls _EJ0.

Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>

src/vmm/src/devices/acpi/cpu_container.rs

@@ -51,6 +51,8 @@ pub const CPU_CONTAINER_ACPI_SIZE: usize = 0xC;
 
 const CPU_ENABLE_BIT: u8 = 1 << 0;
 const CPU_INSERTING_BIT: u8 = 1 << 1;
+const CPU_REMOVING_BIT: u8 = 1 << 2;
+const CPU_EJECT_BIT: u8 = 1 << 3;
 
 const CPU_SELECTION_OFFSET: u64 = 0;
 const CPU_STATUS_OFFSET: u64 = 4;
@@ -94,6 +96,7 @@ impl CpuContainer {
  cpu_id: i,
  active: i < boot_count,
  inserting: false,
+ removing: false,
  })
  }
 
@@ -123,6 +126,9 @@ impl CpuContainer {
  if cpu_device.inserting {
  data[0] |= CPU_INSERTING_BIT;
  }
+ if cpu_device.removing {
+ data[0] |= CPU_REMOVING_BIT;
+ }
  } else {
  error!("Out of range vCPU id: {}", self.selected_cpu)
  }
@@ -143,6 +149,10 @@ impl CpuContainer {
  if data[0] & CPU_ENABLE_BIT != 0 {
  cpu_device.active = true;
  }
+ if data[0] & CPU_EJECT_BIT != 0 {
+ cpu_device.active = false;
+ // TODO: Remove vCPU handle from VMM
+ }
  } else {
  error!("Out of range vCPU id: {}", self.selected_cpu)
  }
@@ -215,7 +225,9 @@ impl Aml for CpuContainer {
  aml::FieldEntry::Reserved(32),
  aml::FieldEntry::Named(*b"CPEN", 1),
  aml::FieldEntry::Named(*b"CINS", 1),
- aml::FieldEntry::Reserved(6),
+ aml::FieldEntry::Named(*b"CRMV", 1),
+ aml::FieldEntry::Named(*b"CEJ0", 1),
+ aml::FieldEntry::Reserved(4),
  aml::FieldEntry::Named(*b"CCMD", 8),
  ],
  ),
@@ -270,6 +282,8 @@ pub struct CpuDevice {
  pub active: bool,
  /// Whether this CPU is in the process of being inserted
  pub inserting: bool,
+ /// Whether this CPU is in the process of being removed
+ pub removing: bool,
 }
 
 impl CpuDevice {
@@ -305,6 +319,16 @@ impl Aml for CpuDevice {
  ))],
  ),
  &aml::Name::new("_MAT".into(), &aml::Buffer::new(mat_data)),
+ &aml::Method::new(
+ "_EJ0".into(),
+ 1,
+ false,
+ // Call into CEJ0 method which will actually eject device
+ vec![&aml::MethodCall::new(
+ "\\_SB_.CPUS.CEJ0".into(),
+ vec![&self.cpu_id],
+ )],
+ ),
  ],
  )
  .append_aml_bytes(v)
@@ -320,7 +344,7 @@ impl Aml for CpuNotify {
  let object = aml::Path::new(&format!("C{:03X}", self.cpu_id));
  aml::If::new(
  &aml::Equal::new(&aml::Arg(0), &self.cpu_id),
- vec![&aml::Notify::new(&object, &1u8)],
+ vec![&aml::Notify::new(&object, &aml::Arg(1))],
  )
  .append_aml_bytes(v)
  }
@@ -369,6 +393,21 @@ impl Aml for CpuMethods {
 
  aml::Method::new("CTFY".into(), 2, true, cpu_notifies_refs).append_aml_bytes(v);
 
+ aml::Method::new(
+ "CEJ0".into(),
+ 1,
+ true,
+ vec![
+ &aml::Acquire::new("\\_SB_.PRES.CPLK".into(), 0xffff),
+ // Write CPU number (in first argument) to I/O port via field
+ &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CSEL"), &aml::Arg(0)),
+ // Set CEJ0 bit
+ &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CEJ0"), &aml::ONE),
+ &aml::Release::new("\\_SB_.PRES.CPLK".into()),
+ ],
+ )
+ .append_aml_bytes(v);
+
  aml::Method::new(
  "CSCN".into(),
  0,
@@ -396,6 +435,15 @@ impl Aml for CpuMethods {
  &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CINS"), &aml::ONE),
  ],
  ),
+ &aml::If::new(
+ &aml::Equal::new(&aml::Path::new("\\_SB_.PRES.CRMV"), &aml::ONE),
+ // Notify device if it is (with the eject constant 0x3)
+ vec![
+ &aml::MethodCall::new("CTFY".into(), vec![&aml::Local(0), &3u8]),
+ // Reset CRMV bit
+ &aml::Store::new(&aml::Path::new("\\_SB_.PRES.CRMV"), &aml::ONE),
+ ],
+ ),
  &aml::Add::new(&aml::Local(0), &aml::Local(0), &aml::ONE),
  ],
  ),

src/vmm/src/lib.rs

@@ -615,7 +615,7 @@ impl Vmm {
  config: HotplugVcpuConfig,
  ) -> Result<MachineConfigUpdate, HotplugVcpuError> {
  use crate::logger::IncMetric;
- if config.target > MAX_SUPPORTED_VCPUS.into() {
+ if config.target > MAX_SUPPORTED_VCPUS {
  return Err(HotplugVcpuError::VcpuCountTooHigh);
  }
 
@@ -682,6 +682,52 @@ impl Vmm {
  }
 
  /// Removes vCPUs from VMM.
+ #[cfg(target_arch = "x86_64")]
+ pub fn hotunplug_vcpus(
+ &mut self,
+ config: HotplugVcpuConfig,
+ ) -> Result<MachineConfigUpdate, HotplugVcpuError> {
+ use crate::logger::IncMetric;
+ if config.target < 1 {
+ return Err(HotplugVcpuError::VcpuCountTooLow);
+ }
+ if let Some(kvm_config) = self.vcpu_config.as_mut() {
+ kvm_config.vcpu_count = config.target;
+ }
+
+ #[allow(clippy::cast_possible_truncation)]
+ let start_idx: u8 = config.target;
+ if let Some(devices::BusDevice::CpuContainer(cont)) =
+ self.get_bus_device(DeviceType::CpuContainer, "CpuContainer")
+ {
+ let mut locked_container = cont.lock().expect("Poisoned lock");
+ for cpu_idx in start_idx..u8::try_from(self.vcpus_handles.len()).unwrap() {
+ locked_container.cpu_devices[cpu_idx as usize].removing = true;
+ }
+ }
+
+ #[allow(clippy::cast_lossless)]
+ METRICS
+ .hotplug
+ .vcpus_added
+ .add(self.vcpus_handles.len() as u64 - config.target as u64);
+
+ // Update VM config to reflect new CPUs added
+ #[allow(clippy::cast_possible_truncation)]
+ let new_machine_config = MachineConfigUpdate {
+ vcpu_count: Some(self.vcpus_handles.len() as u8),
+ mem_size_mib: None,
+ smt: None,
+ cpu_template: None,
+ track_dirty_pages: None,
+ huge_pages: None,
+ };
+
+ self.acpi_device_manager.notify_cpu_container()?;
+
+ Ok(new_machine_config)
+ }
+
  /// Retrieves the KVM dirty bitmap for each of the guest's memory regions.
  pub fn reset_dirty_bitmap(&self) {
  self.guest_memory

src/vmm/src/rpc_interface.rs

@@ -670,7 +670,17 @@ impl RuntimeApiController {
  self.vmm.lock().expect("Poisoned lock").version(),
  )),
  #[cfg(target_arch = "x86_64")]
- HotplugRequest(request_type) => self.handle_hotplug_request(request_type),
+ HotplugRequest(request_type) => {
+ let curr_vcpus: u8 = self
+ .vmm
+ .lock()
+ .expect("Poisoned lock")
+ .vcpus_handles
+ .len()
+ .try_into()
+ .unwrap();
+ self.handle_hotplug_request(request_type, curr_vcpus)
+ }
  PatchMMDS(value) => self.patch_mmds(value),
  Pause => self.pause(),
  PutMMDS(value) => self.put_mmds(value),
@@ -872,13 +882,25 @@ impl RuntimeApiController {
  fn handle_hotplug_request(
  &mut self,
  cfg: HotplugRequestConfig,
+ curr_vcpus: u8,
  ) -> Result<VmmData, VmmActionError> {
  match cfg {
  HotplugRequestConfig::Vcpu(cfg) => {
- let result = self.vmm.lock().expect("Poisoned lock").hotplug_vcpus(cfg);
- result
- .map_err(|err| VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err)))
- .and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update))
+ if cfg.target > curr_vcpus {
+ let result = self.vmm.lock().expect("Poisoned lock").hotplug_vcpus(cfg);
+ result
+ .map_err(|err| {
+ VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err))
+ })
+ .and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update))
+ } else {
+ let result = self.vmm.lock().expect("Poisoned lock").hotunplug_vcpus(cfg);
+ result
+ .map_err(|err| {
+ VmmActionError::HotplugRequest(HotplugRequestError::Vcpu(err))
+ })
+ .and_then(|machine_cfg_update| self.update_vm_config(machine_cfg_update))
+ }
  }
  }
  }