diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index 591f086c44c..06d89156a00 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -40,6 +40,8 @@ mod filesystems; pub(crate) mod ops; mod overlay_fs; pub mod pipe; +#[cfg(feature = "host-fs")] +mod scoped_directory_fs; mod static_file; #[cfg(feature = "static-fs")] pub mod static_fs; @@ -65,6 +67,8 @@ pub use null_file::*; pub use overlay_fs::OverlayFileSystem; pub use passthru_fs::*; pub use pipe::*; +#[cfg(feature = "host-fs")] +pub use scoped_directory_fs::ScopedDirectoryFileSystem; pub use special_file::*; pub use static_file::StaticFile; pub use tmp_fs::*; @@ -672,6 +676,13 @@ impl FileType { } } + pub fn new_file() -> Self { + Self { + file: true, + ..Default::default() + } + } + pub fn is_dir(&self) -> bool { self.dir } diff --git a/lib/virtual-fs/src/scoped_directory_fs.rs b/lib/virtual-fs/src/scoped_directory_fs.rs new file mode 100644 index 00000000000..43b3b020093 --- /dev/null +++ b/lib/virtual-fs/src/scoped_directory_fs.rs @@ -0,0 +1,198 @@ +use std::path::{Component, Path, PathBuf}; + +use futures::future::BoxFuture; + +use crate::{ + DirEntry, FileOpener, FileSystem, FsError, Metadata, OpenOptions, OpenOptionsConfig, ReadDir, + VirtualFile, +}; + +/// A [`FileSystem`] implementation that is scoped to a specific directory on +/// the host. +#[derive(Debug, Clone)] +pub struct ScopedDirectoryFileSystem { + root: PathBuf, + inner: crate::host_fs::FileSystem, +} + +impl ScopedDirectoryFileSystem { + pub fn new(root: impl Into, inner: crate::host_fs::FileSystem) -> Self { + ScopedDirectoryFileSystem { + root: root.into(), + inner, + } + } + + /// Create a new [`ScopedDirectoryFileSystem`] using the current + /// [`tokio::runtime::Handle`]. + /// + /// # Panics + /// + /// This will panic if called outside of a `tokio` context. + pub fn new_with_default_runtime(root: impl Into) -> Self { + let handle = tokio::runtime::Handle::current(); + let fs = crate::host_fs::FileSystem::new(handle); + ScopedDirectoryFileSystem::new(root, fs) + } + + fn prepare_path(&self, path: &Path) -> PathBuf { + let path = normalize_path(path); + let path = path.strip_prefix("/").unwrap_or(&path); + + let path = if !path.starts_with(&self.root) { + self.root.join(path) + } else { + path.to_owned() + }; + + debug_assert!(path.starts_with(&self.root)); + path + } +} + +impl FileSystem for ScopedDirectoryFileSystem { + fn read_dir(&self, path: &Path) -> Result { + let path = self.prepare_path(path); + + let mut entries = Vec::new(); + + for entry in self.inner.read_dir(&path)? { + let entry = entry?; + let path = entry + .path + .strip_prefix(&self.root) + .map_err(|_| FsError::InvalidData)?; + entries.push(DirEntry { + path: Path::new("/").join(path), + ..entry + }); + } + + Ok(ReadDir::new(entries)) + } + + fn create_dir(&self, path: &Path) -> Result<(), FsError> { + let path = self.prepare_path(path); + self.inner.create_dir(&path) + } + + fn remove_dir(&self, path: &Path) -> Result<(), FsError> { + let path = self.prepare_path(path); + self.inner.remove_dir(&path) + } + + fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, Result<(), FsError>> { + Box::pin(async move { + let from = self.prepare_path(from); + let to = self.prepare_path(to); + self.inner.rename(&from, &to).await + }) + } + + fn metadata(&self, path: &Path) -> Result { + let path = self.prepare_path(path); + self.inner.metadata(&path) + } + + fn remove_file(&self, path: &Path) -> Result<(), FsError> { + let path = self.prepare_path(path); + self.inner.remove_file(&path) + } + + fn new_open_options(&self) -> OpenOptions { + OpenOptions::new(self) + } +} + +impl FileOpener for ScopedDirectoryFileSystem { + fn open( + &self, + path: &Path, + conf: &OpenOptionsConfig, + ) -> Result, FsError> { + let path = self.prepare_path(path); + self.inner + .new_open_options() + .options(conf.clone()) + .open(&path) + } +} + +// Copied from cargo +// https://github.com/rust-lang/cargo/blob/fede83ccf973457de319ba6fa0e36ead454d2e20/src/cargo/util/paths.rs#L61 +fn normalize_path(path: &Path) -> PathBuf { + let mut components = path.components().peekable(); + + if matches!(components.peek(), Some(Component::Prefix(..))) { + // This bit diverges from the original cargo implementation, but we want + // to ignore the drive letter or UNC prefix on Windows. This shouldn't + // make a difference in practice because WASI is meant to give us + // Unix-style paths, not Windows-style ones. + let _ = components.next(); + } + + let mut ret = PathBuf::new(); + + for component in components { + match component { + Component::Prefix(..) => unreachable!(), + Component::RootDir => {} + Component::CurDir => {} + Component::ParentDir => { + ret.pop(); + } + Component::Normal(c) => { + ret.push(c); + } + } + } + ret +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + use tokio::io::AsyncReadExt; + + use super::*; + + #[tokio::test] + async fn open_files() { + let temp = TempDir::new().unwrap(); + std::fs::write(temp.path().join("file.txt"), "Hello, World!").unwrap(); + let fs = ScopedDirectoryFileSystem::new_with_default_runtime(temp.path()); + + let mut f = fs.new_open_options().read(true).open("/file.txt").unwrap(); + let mut contents = String::new(); + f.read_to_string(&mut contents).await.unwrap(); + + assert_eq!(contents, "Hello, World!"); + } + + #[tokio::test] + async fn cant_access_outside_the_scoped_directory() { + let scoped_directory = TempDir::new().unwrap(); + std::fs::write(scoped_directory.path().join("file.txt"), "").unwrap(); + std::fs::create_dir_all(scoped_directory.path().join("nested").join("dir")).unwrap(); + let fs = ScopedDirectoryFileSystem::new_with_default_runtime(scoped_directory.path()); + + // Using ".." shouldn't let you escape the scoped directory + let mut directory_entries: Vec<_> = fs + .read_dir("/../../../".as_ref()) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + directory_entries.sort(); + assert_eq!( + directory_entries, + vec![PathBuf::from("/file.txt"), PathBuf::from("/nested")], + ); + + // Using a directory's absolute path also shouldn't work + let other_dir = TempDir::new().unwrap(); + assert_eq!( + fs.read_dir(other_dir.path()).unwrap_err(), + FsError::EntryNotFound + ); + } +} diff --git a/lib/wasix/src/runners/mod.rs b/lib/wasix/src/runners/mod.rs index e32221049ec..fa64d05a8cd 100644 --- a/lib/wasix/src/runners/mod.rs +++ b/lib/wasix/src/runners/mod.rs @@ -9,15 +9,7 @@ mod wasi_common; #[cfg(feature = "webc_runner_rt_wcgi")] pub mod wcgi; -pub use self::{runner::Runner, wasi_common::MappedCommand}; - -/// A directory that should be mapped from the host filesystem into a WASI -/// instance (the "guest"). -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct MappedDirectory { - /// The absolute path for a directory on the host filesystem. - pub host: std::path::PathBuf, - /// The absolute path specifying where the host directory should be mounted - /// inside the guest. - pub guest: String, -} +pub use self::{ + runner::Runner, + wasi_common::{MappedCommand, MappedDirectory, MountedDirectory}, +}; diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 93911957bb7..d21bfaee71f 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -4,7 +4,7 @@ use std::{path::PathBuf, sync::Arc}; use anyhow::{Context, Error}; use tracing::Instrument; -use virtual_fs::{ArcBoxFile, TmpFileSystem, VirtualFile}; +use virtual_fs::{ArcBoxFile, FileSystem, TmpFileSystem, VirtualFile}; use wasmer::Module; use webc::metadata::{annotations::Wasi, Command}; @@ -12,7 +12,7 @@ use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, journal::{DynJournal, SnapshotTrigger}, - runners::{wasi_common::CommonWasiOptions, MappedDirectory}, + runners::{wasi_common::CommonWasiOptions, MappedDirectory, MountedDirectory}, runtime::{module_cache::ModuleHash, task_manager::VirtualTaskManagerExt}, Runtime, WasiEnvBuilder, WasiError, WasiRuntimeError, }; @@ -98,14 +98,25 @@ impl WasiRunner { self.wasi.forward_host_env = forward; } - pub fn with_mapped_directories(mut self, dirs: I) -> Self + pub fn with_mapped_directories(self, dirs: I) -> Self where I: IntoIterator, D: Into, { - self.wasi - .mapped_dirs - .extend(dirs.into_iter().map(|d| d.into())); + self.with_mounted_directories(dirs.into_iter().map(Into::into).map(MountedDirectory::from)) + } + + pub fn with_mounted_directories(mut self, dirs: I) -> Self + where + I: IntoIterator, + D: Into, + { + self.wasi.mounts.extend(dirs.into_iter().map(Into::into)); + self + } + + pub fn mount(&mut self, dest: String, fs: Arc) -> &mut Self { + self.wasi.mounts.push(MountedDirectory { guest: dest, fs }); self } diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 453fa3c077c..6fddde6a6d3 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -14,7 +14,6 @@ use crate::{ bin_factory::BinaryPackage, capabilities::Capabilities, journal::{DynJournal, SnapshotTrigger}, - runners::MappedDirectory, WasiEnvBuilder, }; @@ -32,8 +31,8 @@ pub(crate) struct CommonWasiOptions { pub(crate) args: Vec, pub(crate) env: HashMap, pub(crate) forward_host_env: bool, - pub(crate) mapped_dirs: Vec, pub(crate) mapped_host_commands: Vec, + pub(crate) mounts: Vec, pub(crate) injected_packages: Vec, pub(crate) capabilities: Capabilities, #[derivative(Debug = "ignore")] @@ -52,12 +51,11 @@ impl CommonWasiOptions { root_fs: Option, ) -> Result<(), anyhow::Error> { let root_fs = root_fs.unwrap_or_else(|| RootFileSystemBuilder::default().build()); - - let fs = prepare_filesystem(root_fs, &self.mapped_dirs, container_fs, builder)?; + let fs = prepare_filesystem(root_fs, &self.mounts, container_fs)?; builder.add_preopen_dir("/")?; - if self.mapped_dirs.iter().all(|m| m.guest != ".") { + if self.mounts.iter().all(|m| m.guest != ".") { // The user hasn't mounted "." to anything, so let's map it to "/" builder.add_map_dir(".", "/")?; } @@ -117,31 +115,24 @@ impl CommonWasiOptions { // OverlayFileSystem>; 1]>; fn build_directory_mappings( - builder: &mut WasiEnvBuilder, root_fs: &mut TmpFileSystem, - host_fs: &Arc, - mapped_dirs: &[MappedDirectory], + mounted_dirs: &[MountedDirectory], ) -> Result<(), anyhow::Error> { - for dir in mapped_dirs { - let MappedDirectory { - host: host_path, + for dir in mounted_dirs { + let MountedDirectory { guest: guest_path, + fs, } = dir; let mut guest_path = PathBuf::from(guest_path); tracing::debug!( guest=%guest_path.display(), - host=%host_path.display(), - "Mounting host folder", + "Mounting", ); if guest_path.is_relative() { guest_path = apply_relative_path_mounting_hack(&guest_path); } - let host_path = std::fs::canonicalize(host_path).with_context(|| { - format!("Unable to canonicalize host path '{}'", host_path.display()) - })?; - let guest_path = root_fs .canonicalize_unchecked(&guest_path) .with_context(|| { @@ -153,28 +144,18 @@ fn build_directory_mappings( if guest_path == Path::new("/") { root_fs - .mount_directory_entries(&guest_path, host_fs, &host_path) - .with_context(|| format!("Unable to mount \"{}\" to root", host_path.display(),))?; + .mount_directory_entries(&guest_path, fs, "/".as_ref()) + .context("Unable to mount to root")?; } else { if let Some(parent) = guest_path.parent() { - create_dir_all(root_fs, parent).with_context(|| { + create_dir_all(&*root_fs, parent).with_context(|| { format!("Unable to create the \"{}\" directory", parent.display()) })?; } root_fs - .mount(guest_path.clone(), host_fs, host_path.clone()) - .with_context(|| { - format!( - "Unable to mount \"{}\" to \"{}\"", - host_path.display(), - guest_path.display() - ) - })?; - - builder - .add_preopen_dir(&guest_path) - .with_context(|| format!("Unable to preopen \"{}\"", guest_path.display()))?; + .mount(guest_path.clone(), fs, "/".into()) + .with_context(|| format!("Unable to mount \"{}\"", guest_path.display()))?; } } @@ -183,13 +164,11 @@ fn build_directory_mappings( fn prepare_filesystem( mut root_fs: TmpFileSystem, - mapped_dirs: &[MappedDirectory], + mounted_dirs: &[MountedDirectory], container_fs: Option>, - builder: &mut WasiEnvBuilder, ) -> Result, Error> { - if !mapped_dirs.is_empty() { - let host_fs: Arc = Arc::new(crate::default_fs_backing()); - build_directory_mappings(builder, &mut root_fs, &host_fs, mapped_dirs)?; + if !mounted_dirs.is_empty() { + build_directory_mappings(&mut root_fs, mounted_dirs)?; } // HACK(Michael-F-Bryan): The WebcVolumeFileSystem only accepts relative @@ -257,6 +236,45 @@ fn create_dir_all(fs: &dyn FileSystem, path: &Path) -> Result<(), Error> { Ok(()) } +#[derive(Debug, Clone)] +pub struct MountedDirectory { + pub guest: String, + pub fs: Arc, +} + +/// A directory that should be mapped from the host filesystem into a WASI +/// instance (the "guest"). +/// +/// # Panics +/// +/// Converting a [`MappedDirectory`] to a [`MountedDirectory`] requires enabling +/// the `host-fs` feature flag. Using the [`From`] implementation without +/// enabling this feature will result in a runtime panic. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct MappedDirectory { + /// The absolute path for a directory on the host filesystem. + pub host: std::path::PathBuf, + /// The absolute path specifying where the host directory should be mounted + /// inside the guest. + pub guest: String, +} + +impl From for MountedDirectory { + fn from(value: MappedDirectory) -> Self { + cfg_if::cfg_if! { + if #[cfg(feature = "host-fs")] { + let MappedDirectory { host, guest } = value; + let fs: Arc = + Arc::new(virtual_fs::ScopedDirectoryFileSystem::new_with_default_runtime(host)); + + MountedDirectory { guest, fs } + } else { + unreachable!("The `host-fs` feature needs to be enabled to map {value:?}") + } + } + } +} + #[derive(Debug)] struct RelativeOrAbsolutePathHack(F); @@ -325,9 +343,10 @@ impl virtual_fs::FileOpener for RelativeOrAbsolutePathHack { #[cfg(test)] mod tests { - use tempfile::TempDir; + use std::time::SystemTime; - use virtual_fs::WebcVolumeFileSystem; + use tempfile::TempDir; + use virtual_fs::{DirEntry, FileType, Metadata, WebcVolumeFileSystem}; use webc::Container; use super::*; @@ -395,22 +414,21 @@ mod tests { } #[tokio::test] + #[cfg_attr(not(feature = "host-fs"), ignore)] async fn python_use_case() { let temp = TempDir::new().unwrap(); let sub_dir = temp.path().join("path").join("to"); std::fs::create_dir_all(&sub_dir).unwrap(); std::fs::write(sub_dir.join("file.txt"), b"Hello, World!").unwrap(); - let mapping = [MappedDirectory { + let mapping = [MountedDirectory::from(MappedDirectory { guest: "/home".to_string(), host: sub_dir, - }]; + })]; let container = Container::from_bytes(PYTHON).unwrap(); let webc_fs = WebcVolumeFileSystem::mount_all(&container); - let mut builder = WasiEnvBuilder::new(""); let root_fs = RootFileSystemBuilder::default().build(); - let fs = - prepare_filesystem(root_fs, &mapping, Some(Arc::new(webc_fs)), &mut builder).unwrap(); + let fs = prepare_filesystem(root_fs, &mapping, Some(Arc::new(webc_fs))).unwrap(); assert!(fs.metadata("/home/file.txt".as_ref()).unwrap().is_file()); assert!(fs.metadata("lib".as_ref()).unwrap().is_dir()); @@ -423,4 +441,59 @@ mod tests { .unwrap() .is_file()); } + + fn unix_timestamp_nanos(instant: SystemTime) -> Option { + let duration = instant.duration_since(SystemTime::UNIX_EPOCH).ok()?; + Some(duration.as_nanos() as u64) + } + + #[tokio::test] + #[cfg_attr(not(feature = "host-fs"), ignore)] + async fn convert_mapped_directory_to_mounted_directory() { + let temp = TempDir::new().unwrap(); + let dir = MappedDirectory { + guest: "/mnt/dir".to_string(), + host: temp.path().to_path_buf(), + }; + let contents = "Hello, World!"; + let file_txt = temp.path().join("file.txt"); + std::fs::write(&file_txt, contents).unwrap(); + let metadata = std::fs::metadata(&file_txt).unwrap(); + + let got = MountedDirectory::from(dir); + + let directory_contents: Vec<_> = got + .fs + .read_dir("/".as_ref()) + .unwrap() + .map(|entry| entry.unwrap()) + .collect(); + assert_eq!( + directory_contents, + vec![DirEntry { + path: PathBuf::from("/file.txt"), + metadata: Ok(Metadata { + ft: FileType::new_file(), + // Note: Some timestamps aren't available on MUSL and will + // default to zero. + accessed: metadata + .accessed() + .ok() + .and_then(unix_timestamp_nanos) + .unwrap_or(0), + created: metadata + .created() + .ok() + .and_then(unix_timestamp_nanos) + .unwrap_or(0), + modified: metadata + .modified() + .ok() + .and_then(unix_timestamp_nanos) + .unwrap_or(0), + len: contents.len() as u64, + }) + }] + ); + } } diff --git a/lib/wasix/src/runners/wcgi/runner.rs b/lib/wasix/src/runners/wcgi/runner.rs index 882e8ade5bf..529181cde07 100644 --- a/lib/wasix/src/runners/wcgi/runner.rs +++ b/lib/wasix/src/runners/wcgi/runner.rs @@ -237,7 +237,7 @@ impl Config { } pub fn map_directory(&mut self, dir: MappedDirectory) -> &mut Self { - self.wasi.mapped_dirs.push(dir); + self.wasi.mounts.push(dir.into()); self } @@ -245,7 +245,9 @@ impl Config { &mut self, mappings: impl IntoIterator, ) -> &mut Self { - self.wasi.mapped_dirs.extend(mappings.into_iter()); + for mapping in mappings { + self.map_directory(mapping); + } self }