From a9941e4c29a5a0d77ea6a98b32e8d84b7ae7a441 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 9 Nov 2023 22:44:39 +0800 Subject: [PATCH 1/7] Updated `WasiRunner` to allow mounting `FileSystem` instances as well as directories on the host system --- lib/virtual-fs/src/lib.rs | 7 + lib/wasix/src/runners/mod.rs | 16 +- lib/wasix/src/runners/wasi.rs | 23 ++- lib/wasix/src/runners/wasi_common.rs | 237 ++++++++++++++++++++++----- lib/wasix/src/runners/wcgi/runner.rs | 6 +- 5 files changed, 224 insertions(+), 65 deletions(-) diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index 591f086c44c..fd9775b7998 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -672,6 +672,13 @@ impl FileType { } } + pub fn new_file() -> Self { + Self { + file: true, + ..Default::default() + } + } + pub fn is_dir(&self) -> bool { self.dir } diff --git a/lib/wasix/src/runners/mod.rs b/lib/wasix/src/runners/mod.rs index e32221049ec..fa64d05a8cd 100644 --- a/lib/wasix/src/runners/mod.rs +++ b/lib/wasix/src/runners/mod.rs @@ -9,15 +9,7 @@ mod wasi_common; #[cfg(feature = "webc_runner_rt_wcgi")] pub mod wcgi; -pub use self::{runner::Runner, wasi_common::MappedCommand}; - -/// A directory that should be mapped from the host filesystem into a WASI -/// instance (the "guest"). -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct MappedDirectory { - /// The absolute path for a directory on the host filesystem. - pub host: std::path::PathBuf, - /// The absolute path specifying where the host directory should be mounted - /// inside the guest. - pub guest: String, -} +pub use self::{ + runner::Runner, + wasi_common::{MappedCommand, MappedDirectory, MountedDirectory}, +}; diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 93911957bb7..d21bfaee71f 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -4,7 +4,7 @@ use std::{path::PathBuf, sync::Arc}; use anyhow::{Context, Error}; use tracing::Instrument; -use virtual_fs::{ArcBoxFile, TmpFileSystem, VirtualFile}; +use virtual_fs::{ArcBoxFile, FileSystem, TmpFileSystem, VirtualFile}; use wasmer::Module; use webc::metadata::{annotations::Wasi, Command}; @@ -12,7 +12,7 @@ use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, journal::{DynJournal, SnapshotTrigger}, - runners::{wasi_common::CommonWasiOptions, MappedDirectory}, + runners::{wasi_common::CommonWasiOptions, MappedDirectory, MountedDirectory}, runtime::{module_cache::ModuleHash, task_manager::VirtualTaskManagerExt}, Runtime, WasiEnvBuilder, WasiError, WasiRuntimeError, }; @@ -98,14 +98,25 @@ impl WasiRunner { self.wasi.forward_host_env = forward; } - pub fn with_mapped_directories(mut self, dirs: I) -> Self + pub fn with_mapped_directories(self, dirs: I) -> Self where I: IntoIterator, D: Into, { - self.wasi - .mapped_dirs - .extend(dirs.into_iter().map(|d| d.into())); + self.with_mounted_directories(dirs.into_iter().map(Into::into).map(MountedDirectory::from)) + } + + pub fn with_mounted_directories(mut self, dirs: I) -> Self + where + I: IntoIterator, + D: Into, + { + self.wasi.mounts.extend(dirs.into_iter().map(Into::into)); + self + } + + pub fn mount(&mut self, dest: String, fs: Arc) -> &mut Self { + self.wasi.mounts.push(MountedDirectory { guest: dest, fs }); self } diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 453fa3c077c..a9638723d9d 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -7,14 +7,16 @@ use std::{ use anyhow::{Context, Error}; use derivative::Derivative; use futures::future::BoxFuture; -use virtual_fs::{FileSystem, FsError, OverlayFileSystem, RootFileSystemBuilder, TmpFileSystem}; +use virtual_fs::{ + DirEntry, FileOpener, FileSystem, FsError, OverlayFileSystem, RootFileSystemBuilder, + TmpFileSystem, +}; use webc::metadata::annotations::Wasi as WasiAnnotation; use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, journal::{DynJournal, SnapshotTrigger}, - runners::MappedDirectory, WasiEnvBuilder, }; @@ -32,8 +34,8 @@ pub(crate) struct CommonWasiOptions { pub(crate) args: Vec, pub(crate) env: HashMap, pub(crate) forward_host_env: bool, - pub(crate) mapped_dirs: Vec, pub(crate) mapped_host_commands: Vec, + pub(crate) mounts: Vec, pub(crate) injected_packages: Vec, pub(crate) capabilities: Capabilities, #[derivative(Debug = "ignore")] @@ -52,12 +54,11 @@ impl CommonWasiOptions { root_fs: Option, ) -> Result<(), anyhow::Error> { let root_fs = root_fs.unwrap_or_else(|| RootFileSystemBuilder::default().build()); - - let fs = prepare_filesystem(root_fs, &self.mapped_dirs, container_fs, builder)?; + let fs = prepare_filesystem(root_fs, &self.mounts, container_fs)?; builder.add_preopen_dir("/")?; - if self.mapped_dirs.iter().all(|m| m.guest != ".") { + if self.mounts.iter().all(|m| m.guest != ".") { // The user hasn't mounted "." to anything, so let's map it to "/" builder.add_map_dir(".", "/")?; } @@ -117,31 +118,24 @@ impl CommonWasiOptions { // OverlayFileSystem>; 1]>; fn build_directory_mappings( - builder: &mut WasiEnvBuilder, root_fs: &mut TmpFileSystem, - host_fs: &Arc, - mapped_dirs: &[MappedDirectory], + mounted_dirs: &[MountedDirectory], ) -> Result<(), anyhow::Error> { - for dir in mapped_dirs { - let MappedDirectory { - host: host_path, + for dir in mounted_dirs { + let MountedDirectory { guest: guest_path, + fs, } = dir; let mut guest_path = PathBuf::from(guest_path); tracing::debug!( guest=%guest_path.display(), - host=%host_path.display(), - "Mounting host folder", + "Mounting", ); if guest_path.is_relative() { guest_path = apply_relative_path_mounting_hack(&guest_path); } - let host_path = std::fs::canonicalize(host_path).with_context(|| { - format!("Unable to canonicalize host path '{}'", host_path.display()) - })?; - let guest_path = root_fs .canonicalize_unchecked(&guest_path) .with_context(|| { @@ -153,28 +147,18 @@ fn build_directory_mappings( if guest_path == Path::new("/") { root_fs - .mount_directory_entries(&guest_path, host_fs, &host_path) - .with_context(|| format!("Unable to mount \"{}\" to root", host_path.display(),))?; + .mount_directory_entries(&guest_path, fs, "/".as_ref()) + .context("Unable to mount to root")?; } else { if let Some(parent) = guest_path.parent() { - create_dir_all(root_fs, parent).with_context(|| { + create_dir_all(&*root_fs, parent).with_context(|| { format!("Unable to create the \"{}\" directory", parent.display()) })?; } root_fs - .mount(guest_path.clone(), host_fs, host_path.clone()) - .with_context(|| { - format!( - "Unable to mount \"{}\" to \"{}\"", - host_path.display(), - guest_path.display() - ) - })?; - - builder - .add_preopen_dir(&guest_path) - .with_context(|| format!("Unable to preopen \"{}\"", guest_path.display()))?; + .mount(guest_path.clone(), fs, "/".into()) + .with_context(|| format!("Unable to mount \"{}\"", guest_path.display()))?; } } @@ -183,13 +167,11 @@ fn build_directory_mappings( fn prepare_filesystem( mut root_fs: TmpFileSystem, - mapped_dirs: &[MappedDirectory], + mounted_dirs: &[MountedDirectory], container_fs: Option>, - builder: &mut WasiEnvBuilder, ) -> Result, Error> { - if !mapped_dirs.is_empty() { - let host_fs: Arc = Arc::new(crate::default_fs_backing()); - build_directory_mappings(builder, &mut root_fs, &host_fs, mapped_dirs)?; + if !mounted_dirs.is_empty() { + build_directory_mappings(&mut root_fs, mounted_dirs)?; } // HACK(Michael-F-Bryan): The WebcVolumeFileSystem only accepts relative @@ -257,6 +239,120 @@ fn create_dir_all(fs: &dyn FileSystem, path: &Path) -> Result<(), Error> { Ok(()) } +/// A directory that should be mapped from the host filesystem into a WASI +/// instance (the "guest"). +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct MappedDirectory { + /// The absolute path for a directory on the host filesystem. + pub host: std::path::PathBuf, + /// The absolute path specifying where the host directory should be mounted + /// inside the guest. + pub guest: String, +} + +#[derive(Debug, Clone)] +pub struct MountedDirectory { + pub guest: String, + pub fs: Arc, +} + +impl From for MountedDirectory { + fn from(value: MappedDirectory) -> Self { + let MappedDirectory { host, guest } = value; + let fs: Arc = + Arc::new(HostFolderFileSystem::new(crate::default_fs_backing(), host)); + + MountedDirectory { guest, fs } + } +} + +/// A [`FileSystem`] implementation that is scoped to a specific directory on +/// the host. +#[derive(Debug, Clone)] +struct HostFolderFileSystem { + root: PathBuf, + inner: F, +} + +impl HostFolderFileSystem { + fn new(inner: F, root: PathBuf) -> Self { + HostFolderFileSystem { root, inner } + } + + fn path(&self, path: &Path) -> Result { + let path = path.strip_prefix("/").unwrap_or(path); + Ok(self.root.join(path)) + } +} + +impl FileSystem for HostFolderFileSystem { + fn read_dir(&self, path: &Path) -> virtual_fs::Result { + let path = self.path(path)?; + + let mut entries = Vec::new(); + + for entry in self.inner.read_dir(&path)? { + let entry = entry?; + let path = entry + .path + .strip_prefix(&self.root) + .map_err(|_| FsError::InvalidData)?; + entries.push(DirEntry { + path: Path::new("/").join(path), + ..entry + }); + } + + Ok(virtual_fs::ReadDir::new(entries)) + } + + fn create_dir(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.path(path)?; + self.inner.create_dir(&path) + } + + fn remove_dir(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.path(path)?; + self.inner.remove_dir(&path) + } + + fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, virtual_fs::Result<()>> { + Box::pin(async move { + let from = self.path(from)?; + let to = self.path(to)?; + self.inner.rename(&from, &to).await + }) + } + + fn metadata(&self, path: &Path) -> virtual_fs::Result { + let path = self.path(path)?; + self.inner.metadata(&path) + } + + fn remove_file(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.path(path)?; + self.inner.remove_file(&path) + } + + fn new_open_options(&self) -> virtual_fs::OpenOptions { + virtual_fs::OpenOptions::new(self) + } +} + +impl FileOpener for HostFolderFileSystem { + fn open( + &self, + path: &Path, + conf: &virtual_fs::OpenOptionsConfig, + ) -> virtual_fs::Result> { + let path = self.path(path)?; + self.inner + .new_open_options() + .options(conf.clone()) + .open(&path) + } +} + #[derive(Debug)] struct RelativeOrAbsolutePathHack(F); @@ -325,11 +421,14 @@ impl virtual_fs::FileOpener for RelativeOrAbsolutePathHack { #[cfg(test)] mod tests { - use tempfile::TempDir; + use std::time::SystemTime; - use virtual_fs::WebcVolumeFileSystem; + use tempfile::TempDir; + use virtual_fs::{DirEntry, FileType, Metadata, WebcVolumeFileSystem}; use webc::Container; + use crate::runners::MappedDirectory; + use super::*; const PYTHON: &[u8] = include_bytes!("../../../c-api/examples/assets/python-0.1.0.wasmer"); @@ -400,17 +499,15 @@ mod tests { let sub_dir = temp.path().join("path").join("to"); std::fs::create_dir_all(&sub_dir).unwrap(); std::fs::write(sub_dir.join("file.txt"), b"Hello, World!").unwrap(); - let mapping = [MappedDirectory { + let mapping = [MountedDirectory::from(MappedDirectory { guest: "/home".to_string(), host: sub_dir, - }]; + })]; let container = Container::from_bytes(PYTHON).unwrap(); let webc_fs = WebcVolumeFileSystem::mount_all(&container); - let mut builder = WasiEnvBuilder::new(""); let root_fs = RootFileSystemBuilder::default().build(); - let fs = - prepare_filesystem(root_fs, &mapping, Some(Arc::new(webc_fs)), &mut builder).unwrap(); + let fs = prepare_filesystem(root_fs, &mapping, Some(Arc::new(webc_fs))).unwrap(); assert!(fs.metadata("/home/file.txt".as_ref()).unwrap().is_file()); assert!(fs.metadata("lib".as_ref()).unwrap().is_dir()); @@ -423,4 +520,54 @@ mod tests { .unwrap() .is_file()); } + + #[tokio::test] + async fn convert_mapped_directory_to_mounted_directory() { + let temp = TempDir::new().unwrap(); + let dir = MappedDirectory { + guest: "/mnt/dir".to_string(), + host: temp.path().to_path_buf(), + }; + let contents = "Hello, World!"; + let file_txt = temp.path().join("file.txt"); + std::fs::write(&file_txt, contents).unwrap(); + let metadata = std::fs::metadata(&file_txt).unwrap(); + + let got = MountedDirectory::from(dir); + + let directory_contents: Vec<_> = got + .fs + .read_dir("/".as_ref()) + .unwrap() + .map(|entry| entry.unwrap()) + .collect(); + assert_eq!( + directory_contents, + vec![DirEntry { + path: PathBuf::from("/file.txt"), + metadata: Ok(Metadata { + ft: FileType::new_file(), + accessed: metadata + .accessed() + .unwrap() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64, + created: metadata + .created() + .unwrap() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64, + modified: metadata + .modified() + .unwrap() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64, + len: contents.len() as u64, + }) + }] + ); + } } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 882e8ade5bf..529181cde07 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -237,7 +237,7 @@ impl Config { } pub fn map_directory(&mut self, dir: MappedDirectory) -> &mut Self { - self.wasi.mapped_dirs.push(dir); + self.wasi.mounts.push(dir.into()); self } @@ -245,7 +245,9 @@ impl Config { &mut self, mappings: impl IntoIterator, ) -> &mut Self { - self.wasi.mapped_dirs.extend(mappings.into_iter()); + for mapping in mappings { + self.map_directory(mapping); + } self } From c9eaebec7979b2dbba5345015a3e62d8885b14a2 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 10 Jan 2024 16:58:53 +0800 Subject: [PATCH 2/7] feat: Added a `ScopedDirectoryFileSystem` to the `virtual-fs` crate --- lib/virtual-fs/src/lib.rs | 4 + lib/virtual-fs/src/scoped_directory_fs.rs | 146 ++++++++++++++++++++++ lib/wasix/src/runners/wasi_common.rs | 90 +------------ 3 files changed, 152 insertions(+), 88 deletions(-) create mode 100644 lib/virtual-fs/src/scoped_directory_fs.rs diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index fd9775b7998..06d89156a00 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -40,6 +40,8 @@ mod filesystems; pub(crate) mod ops; mod overlay_fs; pub mod pipe; +#[cfg(feature = "host-fs")] +mod scoped_directory_fs; mod static_file; #[cfg(feature = "static-fs")] pub mod static_fs; @@ -65,6 +67,8 @@ pub use null_file::*; pub use overlay_fs::OverlayFileSystem; pub use passthru_fs::*; pub use pipe::*; +#[cfg(feature = "host-fs")] +pub use scoped_directory_fs::ScopedDirectoryFileSystem; pub use special_file::*; pub use static_file::StaticFile; pub use tmp_fs::*; diff --git a/lib/virtual-fs/src/scoped_directory_fs.rs b/lib/virtual-fs/src/scoped_directory_fs.rs new file mode 100644 index 00000000000..0cdef49bff0 --- /dev/null +++ b/lib/virtual-fs/src/scoped_directory_fs.rs @@ -0,0 +1,146 @@ +use std::path::{Component, Path, PathBuf}; + +use futures::future::BoxFuture; + +use crate::{ + DirEntry, FileOpener, FileSystem, FsError, Metadata, OpenOptions, OpenOptionsConfig, ReadDir, + VirtualFile, +}; + +/// A [`FileSystem`] implementation that is scoped to a specific directory on +/// the host. +#[derive(Debug, Clone)] +pub struct ScopedDirectoryFileSystem { + root: PathBuf, + inner: crate::host_fs::FileSystem, +} + +impl ScopedDirectoryFileSystem { + pub fn new(root: impl Into, inner: crate::host_fs::FileSystem) -> Self { + ScopedDirectoryFileSystem { + root: root.into(), + inner, + } + } + + /// Create a new [`ScopedDirectoryFileSystem`] using the current + /// [`tokio::runtime::Handle`]. + /// + /// # Panics + /// + /// This will panic if called outside of a `tokio` context. + pub fn new_with_default_runtime(root: impl Into) -> Self { + let handle = tokio::runtime::Handle::current(); + let fs = crate::host_fs::FileSystem::new(handle); + ScopedDirectoryFileSystem::new(root, fs) + } + + fn prepare_path(&self, path: &Path) -> PathBuf { + let path = normalize_path(path); + let path = path.strip_prefix("/").unwrap_or(&path); + + let path = if !path.starts_with(&self.root) { + self.root.join(path) + } else { + path.to_owned() + }; + + debug_assert!(path.starts_with(&self.root)); + path + } +} + +impl FileSystem for ScopedDirectoryFileSystem { + fn read_dir(&self, path: &Path) -> Result { + let path = self.prepare_path(path); + + let mut entries = Vec::new(); + + for entry in self.inner.read_dir(&path)? { + let entry = entry?; + let path = entry + .path + .strip_prefix(&self.root) + .map_err(|_| FsError::InvalidData)?; + entries.push(DirEntry { + path: Path::new("/").join(path), + ..entry + }); + } + + Ok(ReadDir::new(entries)) + } + + fn create_dir(&self, path: &Path) -> Result<(), FsError> { + let path = self.prepare_path(path); + self.inner.create_dir(&path) + } + + fn remove_dir(&self, path: &Path) -> Result<(), FsError> { + let path = self.prepare_path(path); + self.inner.remove_dir(&path) + } + + fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, Result<(), FsError>> { + Box::pin(async move { + let from = self.prepare_path(from); + let to = self.prepare_path(to); + self.inner.rename(&from, &to).await + }) + } + + fn metadata(&self, path: &Path) -> Result { + let path = self.prepare_path(path); + self.inner.metadata(&path) + } + + fn remove_file(&self, path: &Path) -> Result<(), FsError> { + let path = self.prepare_path(path); + self.inner.remove_file(&path) + } + + fn new_open_options(&self) -> OpenOptions { + OpenOptions::new(self) + } +} + +impl FileOpener for ScopedDirectoryFileSystem { + fn open( + &self, + path: &Path, + conf: &OpenOptionsConfig, + ) -> Result, FsError> { + let path = self.prepare_path(path); + self.inner + .new_open_options() + .options(conf.clone()) + .open(&path) + } +} + +// Copied from cargo +// https://github.com/rust-lang/cargo/blob/fede83ccf973457de319ba6fa0e36ead454d2e20/src/cargo/util/paths.rs#L61 +fn normalize_path(path: &Path) -> PathBuf { + let mut components = path.components().peekable(); + let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() { + components.next(); + PathBuf::from(c.as_os_str()) + } else { + PathBuf::new() + }; + + for component in components { + match component { + Component::Prefix(..) => unreachable!(), + Component::RootDir => {} + Component::CurDir => {} + Component::ParentDir => { + ret.pop(); + } + Component::Normal(c) => { + ret.push(c); + } + } + } + ret +} diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index a9638723d9d..536eb660a9f 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -9,7 +9,7 @@ use derivative::Derivative; use futures::future::BoxFuture; use virtual_fs::{ DirEntry, FileOpener, FileSystem, FsError, OverlayFileSystem, RootFileSystemBuilder, - TmpFileSystem, + TmpFileSystem, ScopedDirectoryFileSystem, }; use webc::metadata::annotations::Wasi as WasiAnnotation; @@ -260,98 +260,12 @@ impl From for MountedDirectory { fn from(value: MappedDirectory) -> Self { let MappedDirectory { host, guest } = value; let fs: Arc = - Arc::new(HostFolderFileSystem::new(crate::default_fs_backing(), host)); + Arc::new(ScopedDirectoryFileSystem::new_with_default_runtime(host)); MountedDirectory { guest, fs } } } -/// A [`FileSystem`] implementation that is scoped to a specific directory on -/// the host. -#[derive(Debug, Clone)] -struct HostFolderFileSystem { - root: PathBuf, - inner: F, -} - -impl HostFolderFileSystem { - fn new(inner: F, root: PathBuf) -> Self { - HostFolderFileSystem { root, inner } - } - - fn path(&self, path: &Path) -> Result { - let path = path.strip_prefix("/").unwrap_or(path); - Ok(self.root.join(path)) - } -} - -impl FileSystem for HostFolderFileSystem { - fn read_dir(&self, path: &Path) -> virtual_fs::Result { - let path = self.path(path)?; - - let mut entries = Vec::new(); - - for entry in self.inner.read_dir(&path)? { - let entry = entry?; - let path = entry - .path - .strip_prefix(&self.root) - .map_err(|_| FsError::InvalidData)?; - entries.push(DirEntry { - path: Path::new("/").join(path), - ..entry - }); - } - - Ok(virtual_fs::ReadDir::new(entries)) - } - - fn create_dir(&self, path: &Path) -> virtual_fs::Result<()> { - let path = self.path(path)?; - self.inner.create_dir(&path) - } - - fn remove_dir(&self, path: &Path) -> virtual_fs::Result<()> { - let path = self.path(path)?; - self.inner.remove_dir(&path) - } - - fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, virtual_fs::Result<()>> { - Box::pin(async move { - let from = self.path(from)?; - let to = self.path(to)?; - self.inner.rename(&from, &to).await - }) - } - - fn metadata(&self, path: &Path) -> virtual_fs::Result { - let path = self.path(path)?; - self.inner.metadata(&path) - } - - fn remove_file(&self, path: &Path) -> virtual_fs::Result<()> { - let path = self.path(path)?; - self.inner.remove_file(&path) - } - - fn new_open_options(&self) -> virtual_fs::OpenOptions { - virtual_fs::OpenOptions::new(self) - } -} - -impl FileOpener for HostFolderFileSystem { - fn open( - &self, - path: &Path, - conf: &virtual_fs::OpenOptionsConfig, - ) -> virtual_fs::Result> { - let path = self.path(path)?; - self.inner - .new_open_options() - .options(conf.clone()) - .open(&path) - } -} #[derive(Debug)] struct RelativeOrAbsolutePathHack(F); From b9da96a228967cc134dfc115ded1cd3c653c5891 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 10 Jan 2024 17:16:37 +0800 Subject: [PATCH 3/7] Added tests for escaping the scoped directory --- lib/virtual-fs/src/scoped_directory_fs.rs | 48 +++++++++++++++++++++++ lib/wasix/src/runners/wasi_common.rs | 3 +- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/virtual-fs/src/scoped_directory_fs.rs b/lib/virtual-fs/src/scoped_directory_fs.rs index 0cdef49bff0..42cba959704 100644 --- a/lib/virtual-fs/src/scoped_directory_fs.rs +++ b/lib/virtual-fs/src/scoped_directory_fs.rs @@ -144,3 +144,51 @@ fn normalize_path(path: &Path) -> PathBuf { } ret } + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + use tokio::io::AsyncReadExt; + + use super::*; + + #[tokio::test] + async fn open_files() { + let temp = TempDir::new().unwrap(); + std::fs::write(temp.path().join("file.txt"), "Hello, World!").unwrap(); + let fs = ScopedDirectoryFileSystem::new_with_default_runtime(temp.path()); + + let mut f = fs.new_open_options().read(true).open("/file.txt").unwrap(); + let mut contents = String::new(); + f.read_to_string(&mut contents).await.unwrap(); + + assert_eq!(contents, "Hello, World!"); + } + + #[tokio::test] + async fn cant_access_outside_the_scoped_directory() { + let scoped_directory = TempDir::new().unwrap(); + std::fs::write(scoped_directory.path().join("file.txt"), "").unwrap(); + std::fs::create_dir_all(scoped_directory.path().join("nested").join("dir")).unwrap(); + let fs = ScopedDirectoryFileSystem::new_with_default_runtime(scoped_directory.path()); + + // Using ".." shouldn't let you escape the scoped directory + let mut directory_entries: Vec<_> = fs + .read_dir("/../../../".as_ref()) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + directory_entries.sort(); + assert_eq!( + directory_entries, + vec![PathBuf::from("/file.txt"), PathBuf::from("/nested")], + ); + + // Using a directory's absolute path also shouldn't work + let other_dir = TempDir::new().unwrap(); + assert_eq!( + fs.read_dir(other_dir.path()).unwrap_err(), + FsError::EntryNotFound + ); + } +} diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 536eb660a9f..287a994f1fa 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -9,7 +9,7 @@ use derivative::Derivative; use futures::future::BoxFuture; use virtual_fs::{ DirEntry, FileOpener, FileSystem, FsError, OverlayFileSystem, RootFileSystemBuilder, - TmpFileSystem, ScopedDirectoryFileSystem, + ScopedDirectoryFileSystem, TmpFileSystem, }; use webc::metadata::annotations::Wasi as WasiAnnotation; @@ -266,7 +266,6 @@ impl From for MountedDirectory { } } - #[derive(Debug)] struct RelativeOrAbsolutePathHack(F); From a592644b425aa02e68faa500275ad6e6a47c15f7 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 10 Jan 2024 18:11:09 +0800 Subject: [PATCH 4/7] Made sure MountedDirectory still compiles for the browser --- lib/wasix/src/runners/wasi_common.rs | 43 +++++++++++++++++----------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 287a994f1fa..69e1df3eb64 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -7,10 +7,7 @@ use std::{ use anyhow::{Context, Error}; use derivative::Derivative; use futures::future::BoxFuture; -use virtual_fs::{ - DirEntry, FileOpener, FileSystem, FsError, OverlayFileSystem, RootFileSystemBuilder, - ScopedDirectoryFileSystem, TmpFileSystem, -}; +use virtual_fs::{FileSystem, FsError, OverlayFileSystem, RootFileSystemBuilder, TmpFileSystem}; use webc::metadata::annotations::Wasi as WasiAnnotation; use crate::{ @@ -239,8 +236,20 @@ fn create_dir_all(fs: &dyn FileSystem, path: &Path) -> Result<(), Error> { Ok(()) } +#[derive(Debug, Clone)] +pub struct MountedDirectory { + pub guest: String, + pub fs: Arc, +} + /// A directory that should be mapped from the host filesystem into a WASI /// instance (the "guest"). +/// +/// # Panics +/// +/// Converting a [`MappedDirectory`] to a [`MountedDirectory`] requires enabling +/// the `host-fs` feature flag. Using the [`From`] implementation without +/// enabling this feature will result in a runtime panic. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct MappedDirectory { /// The absolute path for a directory on the host filesystem. @@ -250,19 +259,19 @@ pub struct MappedDirectory { pub guest: String, } -#[derive(Debug, Clone)] -pub struct MountedDirectory { - pub guest: String, - pub fs: Arc, -} - impl From for MountedDirectory { fn from(value: MappedDirectory) -> Self { - let MappedDirectory { host, guest } = value; - let fs: Arc = - Arc::new(ScopedDirectoryFileSystem::new_with_default_runtime(host)); - - MountedDirectory { guest, fs } + cfg_if::cfg_if! { + if #[cfg(feature = "host-fs")] { + let MappedDirectory { host, guest } = value; + let fs: Arc = + Arc::new(ScopedDirectoryFileSystem::new_with_default_runtime(host)); + + MountedDirectory { guest, fs } + } else { + unreachable!("The `host-fs` feature needs to be enabled to map {value:?}") + } + } } } @@ -340,8 +349,6 @@ mod tests { use virtual_fs::{DirEntry, FileType, Metadata, WebcVolumeFileSystem}; use webc::Container; - use crate::runners::MappedDirectory; - use super::*; const PYTHON: &[u8] = include_bytes!("../../../c-api/examples/assets/python-0.1.0.wasmer"); @@ -407,6 +414,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(not(feature = "host-fs"), ignore)] async fn python_use_case() { let temp = TempDir::new().unwrap(); let sub_dir = temp.path().join("path").join("to"); @@ -435,6 +443,7 @@ mod tests { } #[tokio::test] + #[cfg_attr(not(feature = "host-fs"), ignore)] async fn convert_mapped_directory_to_mounted_directory() { let temp = TempDir::new().unwrap(); let dir = MappedDirectory { From 962768acbb3158b118c13603353cf28a95492220 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 10 Jan 2024 18:13:01 +0800 Subject: [PATCH 5/7] Forgot an import --- lib/wasix/src/runners/wasi_common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 69e1df3eb64..26eaa638b31 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -265,7 +265,7 @@ impl From for MountedDirectory { if #[cfg(feature = "host-fs")] { let MappedDirectory { host, guest } = value; let fs: Arc = - Arc::new(ScopedDirectoryFileSystem::new_with_default_runtime(host)); + Arc::new(virtual_fs::ScopedDirectoryFileSystem::new_with_default_runtime(host)); MountedDirectory { guest, fs } } else { From ca81d1e6429b9b062de1cf18384a598983750b8f Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 10 Jan 2024 20:23:45 +0800 Subject: [PATCH 6/7] Updated some tests because MUSL doesn't give you access to file creation times --- lib/wasix/src/runners/wasi_common.rs | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 26eaa638b31..6fddde6a6d3 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -442,6 +442,11 @@ mod tests { .is_file()); } + fn unix_timestamp_nanos(instant: SystemTime) -> Option { + let duration = instant.duration_since(SystemTime::UNIX_EPOCH).ok()?; + Some(duration.as_nanos() as u64) + } + #[tokio::test] #[cfg_attr(not(feature = "host-fs"), ignore)] async fn convert_mapped_directory_to_mounted_directory() { @@ -469,24 +474,23 @@ mod tests { path: PathBuf::from("/file.txt"), metadata: Ok(Metadata { ft: FileType::new_file(), + // Note: Some timestamps aren't available on MUSL and will + // default to zero. accessed: metadata .accessed() - .unwrap() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_nanos() as u64, + .ok() + .and_then(unix_timestamp_nanos) + .unwrap_or(0), created: metadata .created() - .unwrap() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_nanos() as u64, + .ok() + .and_then(unix_timestamp_nanos) + .unwrap_or(0), modified: metadata .modified() - .unwrap() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_nanos() as u64, + .ok() + .and_then(unix_timestamp_nanos) + .unwrap_or(0), len: contents.len() as u64, }) }] From 0baaa7e1e2c5559d5a0dc3ffabe66f4d2531e616 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 10 Jan 2024 20:38:07 +0800 Subject: [PATCH 7/7] Made sure normalize_path() handles Windows paths correctly --- lib/virtual-fs/src/scoped_directory_fs.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/virtual-fs/src/scoped_directory_fs.rs b/lib/virtual-fs/src/scoped_directory_fs.rs index 42cba959704..43b3b020093 100644 --- a/lib/virtual-fs/src/scoped_directory_fs.rs +++ b/lib/virtual-fs/src/scoped_directory_fs.rs @@ -122,12 +122,16 @@ impl FileOpener for ScopedDirectoryFileSystem { // https://github.com/rust-lang/cargo/blob/fede83ccf973457de319ba6fa0e36ead454d2e20/src/cargo/util/paths.rs#L61 fn normalize_path(path: &Path) -> PathBuf { let mut components = path.components().peekable(); - let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() { - components.next(); - PathBuf::from(c.as_os_str()) - } else { - PathBuf::new() - }; + + if matches!(components.peek(), Some(Component::Prefix(..))) { + // This bit diverges from the original cargo implementation, but we want + // to ignore the drive letter or UNC prefix on Windows. This shouldn't + // make a difference in practice because WASI is meant to give us + // Unix-style paths, not Windows-style ones. + let _ = components.next(); + } + + let mut ret = PathBuf::new(); for component in components { match component {