Skip to content

Commit

Permalink
feat(vmm): add vhost-user-blk device
Browse files Browse the repository at this point in the history 
This adds vhost-user block device implementation.

In order to create a vhost-user block device,
a PUT /drives request needs to be issued that
contains `vhost-user` object.
At the moment, the object has the only property `socket`
that is a socket path to communicate with
a vhost-user backend.

Example:
```
curl --unix-socket ${fc_socket} -i \
  -X PUT "http://localhost/drives/vhost" \
  -H "accept: application/json" \
  -H "Content-Type: application/json" \
  -d "{
            \"drive_id\": \"vhost\",
            \"is_root_device\": false,
            \"cache_type\": \"Unsafe\",
            \"vhost_user\": {
              \"socket\": \"${vhost_socket}\"
            }
      }"
```

Signed-off-by: Nikita Kalyazin <[email protected]>
Co-authored-by: Diana Popa <[email protected]>
  • Loading branch information
@kalyazin @dianpopa
kalyazin and dianpopa committed Aug 27, 2023
1 parent cc44620 commit 154d48d
Show file tree
Hide file tree
Showing 14 changed files with 760 additions and 33 deletions.
4 changes: 4 additions & 0 deletions resources/seccomp/aarch64-unknown-linux-musl.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -979,6 +979,10 @@
{
"syscall": "sched_yield",
"comment": "Used by the rust standard library in std::sync::mpmc. Firecracker uses mpsc channels from this module for inter-thread communication"
},
{
"syscall": "sendmsg",
"comment": "Used by vhost-user frontend to communicate with the backend"
}
]
}
Expand Down
4 changes: 4 additions & 0 deletions resources/seccomp/x86_64-unknown-linux-musl.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1183,6 +1183,10 @@
{
"syscall": "sched_yield",
"comment": "Used by the rust standard library in std::sync::mpmc. Firecracker uses mpsc channels from this module for inter-thread communication"
},
{
"syscall": "sendmsg",
"comment": "Used by vhost-user frontend to communicate with the backend"
}
]
}
Expand Down
21 changes: 21 additions & 0 deletions src/vmm/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -982,6 +982,26 @@ fn attach_block_devices<'a, I: Iterator<Item = &'a Block> + Debug>(
// The device mutex mustn't be locked here otherwise it will deadlock.
attach_virtio_device(event_manager, vmm, id, block.clone(), cmdline)?;
}
Block::VhostUserBacked(block) => {
let id = {
let locked = block.lock().expect("Poisoned lock");
if locked.is_root_device() {
cmdline.insert_str(if let Some(partuuid) = locked.partuuid() {
format!("root=PARTUUID={}", partuuid)
} else {
// If no PARTUUID was specified for the root device, try with the
// /dev/vda.
"root=/dev/vda".to_string()
})?;

let flags = if locked.is_read_only() { "ro" } else { "rw" };
cmdline.insert_str(flags)?;
}
locked.id().clone()
};
// The device mutex mustn't be locked here otherwise it will deadlock.
attach_virtio_device(event_manager, vmm, id, block.clone(), cmdline)?;
}
}
}
Ok(())
Expand Down Expand Up @@ -1206,6 +1226,7 @@ pub mod tests {
rate_limiter: None,
file_engine_type: FileEngineType::default(),
file: None,
vhost_user: None,
};
block_dev_configs.insert(block_device_config).unwrap();
}
Expand Down
127 changes: 99 additions & 28 deletions src/vmm/src/device_manager/persist.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ use crate::devices::virtio::balloon::persist::{BalloonConstructorArgs, BalloonSt
use crate::devices::virtio::balloon::{Balloon, BalloonError};
use crate::devices::virtio::block::file::persist::{BlockFileConstructorArgs, BlockFileState};
use crate::devices::virtio::block::file::{BlockFile, BlockFileError};
use crate::devices::virtio::block::vhost_user::persist::{
BlockVhostUserConstructorArgs, BlockVhostUserState,
};
use crate::devices::virtio::block::vhost_user::{BlockVhostUser, BlockVhostUserError};
use crate::devices::virtio::net::persist::{
NetConstructorArgs, NetPersistError as NetError, NetState,
};
Expand All @@ -38,8 +42,9 @@ use crate::devices::virtio::vsock::persist::{
};
use crate::devices::virtio::vsock::{Vsock, VsockError, VsockUnixBackend, VsockUnixBackendError};
use crate::devices::virtio::{
MmioTransport, VirtioDevice, SUBTYPE_BALLOON, SUBTYPE_BLOCK_FILE, SUBTYPE_NET, SUBTYPE_RNG,
SUBTYPE_VSOCK, TYPE_BALLOON, TYPE_BLOCK, TYPE_NET, TYPE_RNG, TYPE_VSOCK,
MmioTransport, VirtioDevice, SUBTYPE_BALLOON, SUBTYPE_BLOCK_FILE, SUBTYPE_BLOCK_VHOST_USER,
SUBTYPE_NET, SUBTYPE_RNG, SUBTYPE_VSOCK, TYPE_BALLOON, TYPE_BLOCK, TYPE_NET, TYPE_RNG,
TYPE_VSOCK,
};
use crate::resources::VmResources;
use crate::vmm_config::mmds::MmdsConfigError;
Expand All @@ -50,6 +55,7 @@ use crate::EventManager;
pub enum DevicePersistError {
Balloon(BalloonError),
BlockFile(BlockFileError),
BlockVhostUser(BlockVhostUserError),
DeviceManager(super::mmio::MmioError),
MmioTransport,
#[cfg(target_arch = "aarch64")]
Expand Down Expand Up @@ -89,6 +95,20 @@ pub struct ConnectedBlockFileState {
pub device_info: MMIODeviceInfo,
}

/// Holds the state of a vhost-user-backed block device connected to the MMIO space.
// NOTICE: Any changes to this structure require a snapshot version bump.
#[derive(Debug, Clone, Versionize)]
pub struct ConnectedBlockVhostUserState {
/// Device identifier.
pub device_id: String,
/// Device state.
pub device_state: BlockVhostUserState,
/// Mmio transport state.
pub transport_state: MmioTransportState,
/// VmmResources.
pub device_info: MMIODeviceInfo,
}

/// Holds the state of a net device connected to the MMIO space.
// NOTICE: Any changes to this structure require a snapshot version bump.
#[derive(Debug, Clone, Versionize)]
Expand Down Expand Up @@ -179,6 +199,9 @@ pub struct DeviceStates {
pub block_device_subtypes: Vec<DeviceSubtype>,
/// File-backed block device states.
pub block_file_devices: Vec<ConnectedBlockFileState>,
/// Vhost-user-backed block device states.
#[version(start = 5, de_fn = "de_block_devices", ser_fn = "ser_block_devices")]
pub block_vhost_user_devices: Vec<ConnectedBlockVhostUserState>,
/// Net device states.
pub net_devices: Vec<ConnectedNetState>,
/// Vsock device state.
Expand All @@ -199,6 +222,7 @@ pub struct DeviceStates {
#[derive(Debug)]
pub enum SharedDeviceType {
BlockFile(Arc<Mutex<BlockFile>>),
BlockVhostUser(Arc<Mutex<BlockVhostUser>>),
Network(Arc<Mutex<Net>>),
Balloon(Arc<Mutex<Balloon>>),
Vsock(Arc<Mutex<Vsock<VsockUnixBackend>>>),
Expand Down Expand Up @@ -297,6 +321,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
balloon_device: None,
block_device_subtypes: Vec::new(),
block_file_devices: Vec::new(),
block_vhost_user_devices: Vec::new(),
net_devices: Vec::new(),
vsock_device: None,
#[cfg(target_arch = "aarch64")]
Expand Down Expand Up @@ -347,8 +372,8 @@ impl<'a> Persist<'a> for MMIODeviceManager {
});
}
}
TYPE_BLOCK => {
if locked_device.device_subtype() == SUBTYPE_BLOCK_FILE {
TYPE_BLOCK => match locked_device.device_subtype() {
SUBTYPE_BLOCK_FILE => {
let block = locked_device
.as_mut_any()
.downcast_mut::<BlockFile>()
Expand All @@ -362,7 +387,24 @@ impl<'a> Persist<'a> for MMIODeviceManager {
});
states.block_device_subtypes.push(SUBTYPE_BLOCK_FILE);
}
}
SUBTYPE_BLOCK_VHOST_USER => {
let block = locked_device
.as_mut_any()
.downcast_mut::<BlockVhostUser>()
.unwrap();
block.prepare_save();
states
.block_vhost_user_devices
.push(ConnectedBlockVhostUserState {
device_id: devid.clone(),
device_state: block.save(),
transport_state,
device_info: device_info.clone(),
});
states.block_device_subtypes.push(SUBTYPE_BLOCK_VHOST_USER);
}
_ => (),
},
TYPE_NET => {
if locked_device.device_subtype() == SUBTYPE_NET {
let net = locked_device.as_any().downcast_ref::<Net>().unwrap();
Expand Down Expand Up @@ -557,29 +599,57 @@ impl<'a> Persist<'a> for MMIODeviceManager {
}

let mut block_file_iter = state.block_file_devices.iter();
let mut block_vhost_user_iter = state.block_vhost_user_devices.iter();

for block_device_subtype in &state.block_device_subtypes {
if *block_device_subtype == SUBTYPE_BLOCK_FILE {
// Safe to unwrap, because the subtype vector tracks the device distibution
// between the corresponding vectors.
let block_state = block_file_iter.next().unwrap();
let device = Arc::new(Mutex::new(BlockFile::restore(
BlockFileConstructorArgs { mem: mem.clone() },
&block_state.device_state,
)?));

(constructor_args.for_each_restored_device)(
constructor_args.vm_resources,
SharedDeviceType::BlockFile(device.clone()),
);

restore_helper(
device.clone(),
device,
&block_state.device_id,
&block_state.transport_state,
&block_state.device_info,
constructor_args.event_manager,
)?;
match *block_device_subtype {
SUBTYPE_BLOCK_FILE => {
// Safe to unwrap, because the subtype vector tracks the device distibution
// between the corresponding vectors.
let block_state = block_file_iter.next().unwrap();
let device = Arc::new(Mutex::new(BlockFile::restore(
BlockFileConstructorArgs { mem: mem.clone() },
&block_state.device_state,
)?));

(constructor_args.for_each_restored_device)(
constructor_args.vm_resources,
SharedDeviceType::BlockFile(device.clone()),
);

restore_helper(
device.clone(),
device,
&block_state.device_id,
&block_state.transport_state,
&block_state.device_info,
constructor_args.event_manager,
)?;
}
SUBTYPE_BLOCK_VHOST_USER => {
// Safe to unwrap, because the subtype vector tracks the device distibution
// between the corresponding vectors.
let block_state = block_vhost_user_iter.next().unwrap();
let device = Arc::new(Mutex::new(BlockVhostUser::restore(
BlockVhostUserConstructorArgs { mem: mem.clone() },
&block_state.device_state,
)?));

(constructor_args.for_each_restored_device)(
constructor_args.vm_resources,
SharedDeviceType::BlockVhostUser(device.clone()),
);

restore_helper(
device.clone(),
device,
&block_state.device_id,
&block_state.transport_state,
&block_state.device_info,
constructor_args.event_manager,
)?;
}
_ => (),
}
}

Expand Down Expand Up @@ -936,7 +1006,8 @@ mod tests {
"is_read_only": true,
"rate_limiter": null,
"io_engine": "Sync"
}}
}},
"vhost_user": null
}}
],
"boot-source": {{
Expand Down
1 change: 1 addition & 0 deletions src/vmm/src/devices/virtio/block/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
//! Implements a virtio block device.
pub mod file;
pub mod vhost_user;

use serde::{Deserialize, Serialize};
use versionize::{VersionMap, Versionize, VersionizeError, VersionizeResult};
Expand Down
Loading

0 comments on commit 154d48d

Please sign in to comment.