devices:bus: Add HotPlugBus Trait
Device implement HotPlugBus trait could notify hotplug event into
guest, and such device should be added into RunnableLinuxVm, so it
could be used at device plug in and plug out.
BUG=b:185084350
TEST=Boot a guest with and without passthrough device
Change-Id: I9497f61312582483090ff708d0f37b97d7303811
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2954673
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Daniel Verkamp <dverkamp@chromium.org>
diff --git a/aarch64/src/lib.rs b/aarch64/src/lib.rs
index c25a817..ff2df60 100644
--- a/aarch64/src/lib.rs
+++ b/aarch64/src/lib.rs
@@ -12,7 +12,9 @@
use arch::{get_serial_cmdline, GetSerialCmdlineError, RunnableLinuxVm, VmComponents, VmImage};
use base::{Event, MemoryMappingBuilder};
use devices::serial_device::{SerialHardware, SerialParameters};
-use devices::{Bus, BusError, IrqChip, IrqChipAArch64, PciConfigMmio, PciDevice, ProtectionType};
+use devices::{
+ Bus, BusError, IrqChip, IrqChipAArch64, PciAddress, PciConfigMmio, PciDevice, ProtectionType,
+};
use hypervisor::{DeviceKind, Hypervisor, HypervisorCap, VcpuAArch64, VcpuFeature, VmAArch64};
use minijail::Minijail;
use remain::sorted;
@@ -460,6 +462,7 @@
bat_control: None,
resume_notify_devices: Vec::new(),
root_config: pci_bus,
+ hotplug_bus: None,
})
}
@@ -482,7 +485,7 @@
_device: Box<dyn PciDevice>,
_minijail: Option<Minijail>,
_resources: &mut SystemAllocator,
- ) -> std::result::Result<(), Self::Error> {
+ ) -> std::result::Result<PciAddress, Self::Error> {
// hotplug function isn't verified on AArch64, so set it unsupported here.
Err(Error::Unsupported)
}
diff --git a/arch/src/lib.rs b/arch/src/lib.rs
index 77c6b78..13db64f 100644
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -20,8 +20,9 @@
use base::{syslog, AsRawDescriptor, AsRawDescriptors, Event, Tube};
use devices::virtio::VirtioDevice;
use devices::{
- Bus, BusDevice, BusError, BusResumeDevice, IrqChip, PciAddress, PciDevice, PciDeviceError,
- PciInterruptPin, PciRoot, ProtectionType, ProxyDevice, SerialHardware, SerialParameters,
+ Bus, BusDevice, BusError, BusResumeDevice, HotPlugBus, IrqChip, PciAddress, PciDevice,
+ PciDeviceError, PciInterruptPin, PciRoot, ProtectionType, ProxyDevice, SerialHardware,
+ SerialParameters,
};
use hypervisor::{IoEventAddress, Vm};
use minijail::Minijail;
@@ -121,6 +122,7 @@
/// Devices to be notified before the system resumes from the S3 suspended state.
pub resume_notify_devices: Vec<Arc<Mutex<dyn BusResumeDevice>>>,
pub root_config: Arc<Mutex<RootConfigArch>>,
+ pub hotplug_bus: Option<Arc<Mutex<dyn HotPlugBus>>>,
}
/// The device and optional jail.
@@ -210,7 +212,7 @@
device: Box<dyn PciDevice>,
minijail: Option<Minijail>,
resources: &mut SystemAllocator,
- ) -> Result<(), Self::Error>;
+ ) -> Result<PciAddress, Self::Error>;
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
/// Reads vCPU's registers.
@@ -332,7 +334,7 @@
mut device: Box<dyn PciDevice>,
jail: Option<Minijail>,
resources: &mut SystemAllocator,
-) -> Result<(), DeviceRegistrationError> {
+) -> Result<PciAddress, DeviceRegistrationError> {
// Allocate PCI device address before allocating BARs.
let pci_address = device
.allocate_address(resources)
@@ -418,7 +420,7 @@
.map_err(DeviceRegistrationError::MmioInsert)?;
}
- Ok(())
+ Ok(pci_address)
}
/// Creates a root PCI device for use by this Vm.
diff --git a/devices/src/bus.rs b/devices/src/bus.rs
index 2984937..63c7dc3 100644
--- a/devices/src/bus.rs
+++ b/devices/src/bus.rs
@@ -13,6 +13,8 @@
use serde::{Deserialize, Serialize};
use sync::Mutex;
+use crate::PciAddress;
+
/// Information about how a device was accessed.
#[derive(Copy, Clone, Eq, PartialEq, Debug, Serialize, Deserialize)]
pub struct BusAccessInfo {
@@ -95,6 +97,29 @@
fn resume_imminent(&mut self) {}
}
+/// The key to identify hotplug device from host view.
+/// like host sysfs path for vfio pci device, host disk file
+/// path for virtio block device
+pub enum HostHotPlugKey {
+ Vfio { host_addr: PciAddress },
+}
+
+/// Trait for devices that notify hotplug event into guest
+pub trait HotPlugBus {
+ /// Notify hotplug in event into guest
+ /// * 'addr' - the guest pci address for hotplug in device
+ fn hot_plug(&mut self, addr: PciAddress);
+ /// Notify hotplug out event into guest
+ /// * 'addr' - the guest pci address for hotplug out device
+ fn hot_unplug(&mut self, addr: PciAddress);
+ /// Add hotplug device into this bus
+ /// * 'host_key' - the key to identify hotplug device from host view
+ /// * 'guest_addr' - the guest pci address for hotplug device
+ fn add_hotplug_device(&mut self, host_key: HostHotPlugKey, guest_addr: PciAddress);
+ /// get guest pci address from the specified host_key
+ fn get_hotplug_device(&self, host_key: HostHotPlugKey) -> Option<PciAddress>;
+}
+
#[derive(Debug)]
pub enum Error {
/// The insertion failed because the new device overlapped with an old device.
diff --git a/devices/src/lib.rs b/devices/src/lib.rs
index ffce6a5..0493cf1 100644
--- a/devices/src/lib.rs
+++ b/devices/src/lib.rs
@@ -34,7 +34,10 @@
pub use self::acpi::ACPIPMResource;
pub use self::bat::{BatteryError, GoldfishBattery};
pub use self::bus::Error as BusError;
-pub use self::bus::{Bus, BusAccessInfo, BusDevice, BusDeviceSync, BusRange, BusResumeDevice};
+pub use self::bus::{
+ Bus, BusAccessInfo, BusDevice, BusDeviceSync, BusRange, BusResumeDevice, HostHotPlugKey,
+ HotPlugBus,
+};
pub use self::cmos::Cmos;
#[cfg(feature = "direct")]
pub use self::direct_io::DirectIo;
diff --git a/src/error.rs b/src/error.rs
index cb3d97e..d7aceae 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -42,8 +42,8 @@
ChownTpmStorage(base::Error),
CloneEvent(base::Error),
CloneVcpu(base::Error),
+ ConfigureHotPlugDevice(<Arch as LinuxArch>::Error),
ConfigureVcpu(<Arch as LinuxArch>::Error),
- ConfigureVfioDevice(<Arch as LinuxArch>::Error),
ConnectTube(io::Error),
#[cfg(feature = "audio")]
CreateAc97(devices::PciDeviceError),
@@ -86,11 +86,14 @@
HandleDebugCommand(<Arch as LinuxArch>::Error),
InputDeviceNew(virtio::InputError),
InputEventsOpen(io::Error),
+ InvalidHotPlugKey,
+ InvalidVfioPath,
InvalidWaylandPath,
IoJail(minijail::Error),
LoadKernel(Box<dyn StdError>),
MemoryTooLarge,
NetDeviceNew(virtio::NetError),
+ NoHotPlugBus,
OpenAcpiTable(PathBuf, io::Error),
OpenAndroidFstab(PathBuf, io::Error),
OpenBios(PathBuf, io::Error),
@@ -169,8 +172,8 @@
ChownTpmStorage(e) => write!(f, "failed to chown tpm storage: {}", e),
CloneEvent(e) => write!(f, "failed to clone event: {}", e),
CloneVcpu(e) => write!(f, "failed to clone vcpu: {}", e),
+ ConfigureHotPlugDevice(e) => write!(f, "Failed to configure pci hotplug device:{}", e),
ConfigureVcpu(e) => write!(f, "failed to configure vcpu: {}", e),
- ConfigureVfioDevice(e) => write!(f, "Failed to configure vfio device:{}", e),
ConnectTube(e) => write!(f, "failed to connect to tube: {}", e),
#[cfg(feature = "audio")]
CreateAc97(e) => write!(f, "failed to create ac97 device: {}", e),
@@ -215,11 +218,14 @@
HandleDebugCommand(e) => write!(f, "failed to handle a gdb command: {}", e),
InputDeviceNew(e) => write!(f, "failed to set up input device: {}", e),
InputEventsOpen(e) => write!(f, "failed to open event device: {}", e),
+ InvalidHotPlugKey => write!(f, "failed to find hotplug key in hotplug bus"),
+ InvalidVfioPath => write!(f, "failed to parse or find vfio path"),
InvalidWaylandPath => write!(f, "wayland socket path has no parent or file name"),
IoJail(e) => write!(f, "{}", e),
LoadKernel(e) => write!(f, "failed to load kernel: {}", e),
MemoryTooLarge => write!(f, "requested memory size too large"),
NetDeviceNew(e) => write!(f, "failed to set up virtio networking: {}", e),
+ NoHotPlugBus => write!(f, "HotPlugBus hasn't been implemented"),
OpenAcpiTable(p, e) => write!(f, "failed to open ACPI file {}: {}", p.display(), e),
OpenAndroidFstab(p, e) => write!(
f,
diff --git a/src/linux.rs b/src/linux.rs
index a8be4e9..f588382 100644
--- a/src/linux.rs
+++ b/src/linux.rs
@@ -45,8 +45,8 @@
use devices::Ac97Dev;
use devices::ProtectionType;
use devices::{
- self, IrqChip, IrqEventIndex, KvmKernelIrqChip, PciDevice, VcpuRunState, VfioContainer,
- VfioDevice, VfioPciDevice, VirtioPciDevice,
+ self, HostHotPlugKey, IrqChip, IrqEventIndex, KvmKernelIrqChip, PciAddress, PciDevice,
+ VcpuRunState, VfioContainer, VfioDevice, VfioPciDevice, VirtioPciDevice,
};
#[cfg(feature = "usb")]
use devices::{HostBackendDeviceProvider, XhciController};
@@ -2520,23 +2520,54 @@
control_tubes: &mut Vec<TaggedControlTube>,
vfio_path: &Path,
) -> Result<()> {
- let vm = &linux.vm;
let mut endpoints: BTreeMap<u32, Arc<Mutex<VfioContainer>>> = BTreeMap::new();
let (vfio_pci_device, jail) = create_vfio_device(
cfg,
- vm,
+ &linux.vm,
sys_allocator,
control_tubes,
vfio_path,
&mut endpoints,
false,
)?;
- Arch::register_pci_device(linux, vfio_pci_device, jail, sys_allocator)
- .map_err(Error::ConfigureVfioDevice)
+
+ let pci_address = Arch::register_pci_device(linux, vfio_pci_device, jail, sys_allocator)
+ .map_err(Error::ConfigureHotPlugDevice)?;
+
+ let host_os_str = vfio_path.file_name().ok_or(Error::InvalidVfioPath)?;
+ let host_str = host_os_str.to_str().ok_or(Error::InvalidVfioPath)?;
+ let host_addr = PciAddress::from_string(host_str);
+ let host_key = HostHotPlugKey::Vfio { host_addr };
+ if let Some(hp_bus) = &linux.hotplug_bus {
+ let mut hp_bus = hp_bus.lock();
+ hp_bus.add_hotplug_device(host_key, pci_address);
+ hp_bus.hot_plug(pci_address);
+ return Ok(());
+ }
+
+ Err(Error::NoHotPlugBus)
}
#[allow(dead_code)]
-fn remove_vfio_device() {}
+fn remove_vfio_device<V: VmArch, Vcpu: VcpuArch>(
+ linux: &RunnableLinuxVm<V, Vcpu>,
+ vfio_path: &Path,
+) -> Result<()> {
+ let host_os_str = vfio_path.file_name().ok_or(Error::InvalidVfioPath)?;
+ let host_str = host_os_str.to_str().ok_or(Error::InvalidVfioPath)?;
+ let host_addr = PciAddress::from_string(host_str);
+ let host_key = HostHotPlugKey::Vfio { host_addr };
+ if let Some(hp_bus) = &linux.hotplug_bus {
+ let mut hp_bus = hp_bus.lock();
+ let pci_addr = hp_bus
+ .get_hotplug_device(host_key)
+ .ok_or(Error::InvalidHotPlugKey)?;
+ hp_bus.hot_unplug(pci_addr);
+ return Ok(());
+ }
+
+ Err(Error::NoHotPlugBus)
+}
/// Signals all running VCPUs to vmexit, sends VcpuControl message to each VCPU tube, and tells
/// `irq_chip` to stop blocking halted VCPUs. The channel message is set first because both the
diff --git a/x86_64/src/lib.rs b/x86_64/src/lib.rs
index 40fe364..9780d1c 100644
--- a/x86_64/src/lib.rs
+++ b/x86_64/src/lib.rs
@@ -61,7 +61,9 @@
};
use base::Event;
use devices::serial_device::{SerialHardware, SerialParameters};
-use devices::{BusResumeDevice, IrqChip, IrqChipX86_64, PciConfigIo, PciDevice, ProtectionType};
+use devices::{
+ BusResumeDevice, IrqChip, IrqChipX86_64, PciAddress, PciConfigIo, PciDevice, ProtectionType,
+};
use hypervisor::{HypervisorX86_64, VcpuX86_64, VmX86_64};
use minijail::Minijail;
use remain::sorted;
@@ -556,6 +558,7 @@
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
gdb: components.gdb,
root_config: pci_bus,
+ hotplug_bus: None,
})
}
@@ -600,9 +603,11 @@
device: Box<dyn PciDevice>,
minijail: Option<Minijail>,
resources: &mut SystemAllocator,
- ) -> Result<()> {
- arch::configure_pci_device(linux, device, minijail, resources)
- .map_err(Error::ConfigurePciDevice)
+ ) -> Result<PciAddress> {
+ let pci_address = arch::configure_pci_device(linux, device, minijail, resources)
+ .map_err(Error::ConfigurePciDevice)?;
+
+ Ok(pci_address)
}
#[cfg(all(target_arch = "x86_64", feature = "gdb"))]
