From 4823c0bbf1039ec25d80a55909656eb60841591f Mon Sep 17 00:00:00 2001 From: zjregee Date: Fri, 9 Aug 2024 20:20:59 +0800 Subject: [PATCH 1/2] feat: support getattr and setattr --- integrations/virtiofs/Cargo.toml | 10 +- integrations/virtiofs/src/error.rs | 17 +++ integrations/virtiofs/src/filesystem.rs | 143 +++++++++++++++++- .../virtiofs/src/filesystem_message.rs | 48 ++++++ 4 files changed, 210 insertions(+), 8 deletions(-) diff --git a/integrations/virtiofs/Cargo.toml b/integrations/virtiofs/Cargo.toml index d3eca2361b2..535e8e6112a 100644 --- a/integrations/virtiofs/Cargo.toml +++ b/integrations/virtiofs/Cargo.toml @@ -32,10 +32,12 @@ anyhow = { version = "1.0.86", features = ["std"] } libc = "0.2.139" log = "0.4.22" opendal = { version = "0.48.0", path = "../../core" } -snafu = "0.8.3" -vhost = "0.11.0" -vhost-user-backend = "0.14.0" -virtio-bindings = "0.2.1" +sharded-slab = "0.1.7" +snafu = "0.8.4" +tokio = { version = "1.39.2", features = ["rt-multi-thread"] } +vhost = "0.10.0" +vhost-user-backend = "0.13.1" +virtio-bindings = { version = "0.2.1", features = ["virtio-v5_0_0"] } virtio-queue = "0.11.0" vm-memory = { version = "0.14.0", features = [ "backend-mmap", diff --git a/integrations/virtiofs/src/error.rs b/integrations/virtiofs/src/error.rs index d8c0ff125c8..99d67a86ecf 100644 --- a/integrations/virtiofs/src/error.rs +++ b/integrations/virtiofs/src/error.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::ffi::CStr; use std::io; use anyhow::Error as AnyError; @@ -38,6 +39,22 @@ pub enum Error { }, } +impl From for Error { + fn from(errno: libc::c_int) -> Error { + let err_str = unsafe { libc::strerror(errno) }; + let message = if err_str.is_null() { + format!("errno: {}", errno) + } else { + let c_str = unsafe { CStr::from_ptr(err_str) }; + c_str.to_string_lossy().into_owned() + }; + Error::VhostUserFsError { + message, + source: None, + } + } +} + impl From for io::Error { fn from(error: Error) -> io::Error { match error { diff --git a/integrations/virtiofs/src/filesystem.rs b/integrations/virtiofs/src/filesystem.rs index cdaedea6d02..e65022037c6 100644 --- a/integrations/virtiofs/src/filesystem.rs +++ b/integrations/virtiofs/src/filesystem.rs @@ -17,8 +17,14 @@ use std::io::Write; use std::mem::size_of; +use std::time::Duration; +use log::debug; +use opendal::ErrorKind; use opendal::Operator; +use sharded_slab::Slab; +use tokio::runtime::Builder; +use tokio::runtime::Runtime; use vm_memory::ByteValued; use crate::error::*; @@ -32,21 +38,99 @@ const KERNEL_MINOR_VERSION: u32 = 38; /// Minimum Minor version number supported. const MIN_KERNEL_MINOR_VERSION: u32 = 27; /// The length of the header part of the message. -const BUFFER_HEADER_SIZE: u32 = 256; +const BUFFER_HEADER_SIZE: u32 = 4096; /// The maximum length of the data part of the message, used for read/write data. const MAX_BUFFER_SIZE: u32 = 1 << 20; +/// The deafult time to live of the attributes. +const DEFAULT_TTL: Duration = Duration::from_secs(1); +/// The default mode of the opened file. +const DEFAULT_OPENED_FILE_MODE: u32 = 0o755; + +enum FileType { + Dir, + File, +} + +struct OpenedFile { + path: String, + metadata: Attr, +} + +impl OpenedFile { + fn new(file_type: FileType, path: &str, uid: u32, gid: u32) -> OpenedFile { + let mut attr: Attr = unsafe { std::mem::zeroed() }; + attr.uid = uid; + attr.gid = gid; + match file_type { + FileType::Dir => { + attr.nlink = 2; + attr.mode = libc::S_IFDIR | DEFAULT_OPENED_FILE_MODE; + } + FileType::File => { + attr.nlink = 1; + attr.mode = libc::S_IFREG | DEFAULT_OPENED_FILE_MODE; + } + } + OpenedFile { + path: path.to_string(), + metadata: attr, + } + } +} + +fn opendal_error2error(error: opendal::Error) -> Error { + match error.kind() { + ErrorKind::Unsupported => Error::from(libc::EOPNOTSUPP), + ErrorKind::IsADirectory => Error::from(libc::EISDIR), + ErrorKind::NotFound => Error::from(libc::ENOENT), + ErrorKind::PermissionDenied => Error::from(libc::EACCES), + ErrorKind::AlreadyExists => Error::from(libc::EEXIST), + ErrorKind::NotADirectory => Error::from(libc::ENOTDIR), + ErrorKind::RangeNotSatisfied => Error::from(libc::EINVAL), + ErrorKind::RateLimited => Error::from(libc::EBUSY), + _ => Error::from(libc::ENOENT), + } +} + +fn opendal_metadata2opened_file( + path: &str, + metadata: &opendal::Metadata, + uid: u32, + gid: u32, +) -> OpenedFile { + let file_type = match metadata.mode() { + opendal::EntryMode::DIR => FileType::Dir, + _ => FileType::File, + }; + OpenedFile::new(file_type, path, uid, gid) +} /// Filesystem is a filesystem implementation with opendal backend, /// and will decode and process messages from VMs. pub struct Filesystem { - // FIXME: #[allow(dead_code)] here should be removed in the future. - #[allow(dead_code)] + rt: Runtime, core: Operator, + uid: u32, + gid: u32, + opened_files: Slab, } impl Filesystem { pub fn new(core: Operator) -> Filesystem { - Filesystem { core } + let rt = Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + // Here we set the uid and gid to 1000, which is the default value. + Filesystem { + rt, + core, + uid: 1000, + gid: 1000, + opened_files: Slab::new(), + } } pub fn handle_message(&self, mut r: Reader, w: Writer) -> Result { @@ -60,6 +144,9 @@ impl Filesystem { if let Ok(opcode) = Opcode::try_from(in_header.opcode) { match opcode { Opcode::Init => self.init(in_header, r, w), + Opcode::Destroy => self.destory(in_header, r, w), + Opcode::Getattr => self.getattr(in_header, r, w), + Opcode::Setattr => self.setattr(in_header, r, w), } } else { Filesystem::reply_error(in_header.unique, w) @@ -134,4 +221,52 @@ impl Filesystem { }; Filesystem::reply_ok(Some(out), None, in_header.unique, w) } + + fn destory(&self, _in_header: InHeader, _r: Reader, _w: Writer) -> Result { + // do nothing for destory. + Ok(0) + } + + fn getattr(&self, in_header: InHeader, _r: Reader, w: Writer) -> Result { + debug!("getattr: inode={}", in_header.nodeid); + + let path = match self + .opened_files + .get(in_header.nodeid as usize) + .map(|f| f.path.clone()) + { + Some(path) => path, + None => return Filesystem::reply_error(in_header.unique, w), + }; + + let mut metadata = match self.rt.block_on(self.do_get_metadata(&path)) { + Ok(metadata) => metadata, + Err(_) => return Filesystem::reply_error(in_header.unique, w), + }; + metadata.metadata.ino = in_header.nodeid; + + let out = AttrOut { + attr_valid: DEFAULT_TTL.as_secs(), + attr_valid_nsec: DEFAULT_TTL.subsec_nanos(), + attr: metadata.metadata, + ..Default::default() + }; + Filesystem::reply_ok(Some(out), None, in_header.unique, w) + } + + fn setattr(&self, in_header: InHeader, _r: Reader, w: Writer) -> Result { + debug!("setattr: inode={}", in_header.nodeid); + + // do nothing for setattr. + self.getattr(in_header, _r, w) + } +} + +impl Filesystem { + async fn do_get_metadata(&self, path: &str) -> Result { + let metadata = self.core.stat(path).await.map_err(opendal_error2error)?; + let attr = opendal_metadata2opened_file(path, &metadata, self.uid, self.gid); + + Ok(attr) + } } diff --git a/integrations/virtiofs/src/filesystem_message.rs b/integrations/virtiofs/src/filesystem_message.rs index c427c5c638a..dd924507e45 100644 --- a/integrations/virtiofs/src/filesystem_message.rs +++ b/integrations/virtiofs/src/filesystem_message.rs @@ -23,7 +23,10 @@ use crate::error::*; /// The corresponding value needs to be aligned with the specification. #[non_exhaustive] pub enum Opcode { + Getattr = 3, + Setattr = 4, Init = 26, + Destroy = 38, } impl TryFrom for Opcode { @@ -31,12 +34,41 @@ impl TryFrom for Opcode { fn try_from(value: u32) -> Result { match value { + 3 => Ok(Opcode::Getattr), + 4 => Ok(Opcode::Setattr), 26 => Ok(Opcode::Init), + 38 => Ok(Opcode::Destroy), _ => Err(new_vhost_user_fs_error("failed to decode opcode", None)), } } } +/// Attr represents the file attributes in virtiofs. +/// +/// The fields of the struct need to conform to the specific format of the virtiofs message. +/// Currently, we only need to align them exactly with virtiofsd. +/// Reference: https://gitlab.com/virtio-fs/virtiofsd/-/blob/main/src/fuse.rs?ref_type=heads#L577 +#[repr(C)] +#[derive(Debug, Default, Clone, Copy)] +pub struct Attr { + pub ino: u64, + pub size: u64, + pub blocks: u64, + pub atime: u64, + pub mtime: u64, + pub ctime: u64, + pub atimensec: u32, + pub mtimensec: u32, + pub ctimensec: u32, + pub mode: u32, + pub nlink: u32, + pub uid: u32, + pub gid: u32, + pub rdev: u32, + pub blksize: u32, + pub flags: u32, +} + /// InHeader represents the incoming message header in the filesystem call. /// /// The fields of the struct need to conform to the specific format of the virtiofs message. @@ -105,9 +137,25 @@ pub struct InitOut { pub unused: [u32; 7], } +/// AttrOut is used to return the file attributes in the filesystem call. +/// +/// The fields of the struct need to conform to the specific format of the virtiofs message. +/// Currently, we only need to align them exactly with virtiofsd. +/// Reference: https://gitlab.com/virtio-fs/virtiofsd/-/blob/main/src/fuse.rs?ref_type=heads#L782 +#[repr(C)] +#[derive(Debug, Default, Clone, Copy)] +pub struct AttrOut { + pub attr_valid: u64, + pub attr_valid_nsec: u32, + pub dummy: u32, + pub attr: Attr, +} + /// We will use ByteValued to implement the encoding and decoding /// of these structures in shared memory. +unsafe impl ByteValued for Attr {} unsafe impl ByteValued for InHeader {} unsafe impl ByteValued for OutHeader {} unsafe impl ByteValued for InitIn {} unsafe impl ByteValued for InitOut {} +unsafe impl ByteValued for AttrOut {} From d172418e2978176949ddde05624b2f1527039086 Mon Sep 17 00:00:00 2001 From: zjregee Date: Fri, 9 Aug 2024 20:29:48 +0800 Subject: [PATCH 2/2] typo --- integrations/virtiofs/src/filesystem.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integrations/virtiofs/src/filesystem.rs b/integrations/virtiofs/src/filesystem.rs index e65022037c6..4fb38b15dc2 100644 --- a/integrations/virtiofs/src/filesystem.rs +++ b/integrations/virtiofs/src/filesystem.rs @@ -41,7 +41,7 @@ const MIN_KERNEL_MINOR_VERSION: u32 = 27; const BUFFER_HEADER_SIZE: u32 = 4096; /// The maximum length of the data part of the message, used for read/write data. const MAX_BUFFER_SIZE: u32 = 1 << 20; -/// The deafult time to live of the attributes. +/// The default time to live of the attributes. const DEFAULT_TTL: Duration = Duration::from_secs(1); /// The default mode of the opened file. const DEFAULT_OPENED_FILE_MODE: u32 = 0o755; @@ -144,7 +144,7 @@ impl Filesystem { if let Ok(opcode) = Opcode::try_from(in_header.opcode) { match opcode { Opcode::Init => self.init(in_header, r, w), - Opcode::Destroy => self.destory(in_header, r, w), + Opcode::Destroy => self.destroy(in_header, r, w), Opcode::Getattr => self.getattr(in_header, r, w), Opcode::Setattr => self.setattr(in_header, r, w), } @@ -222,8 +222,8 @@ impl Filesystem { Filesystem::reply_ok(Some(out), None, in_header.unique, w) } - fn destory(&self, _in_header: InHeader, _r: Reader, _w: Writer) -> Result { - // do nothing for destory. + fn destroy(&self, _in_header: InHeader, _r: Reader, _w: Writer) -> Result { + // do nothing for destroy. Ok(0) }