Automatically manage FEX rootfs

This adds a mechanism for automatically managing the FEX rootfs,
mounting and overlayfs'ing the relevant squashfs files from the host (as
many as desired).

Depends on:

 * libkrun containers/libkrun#217
 * libkrunfw containers/libkrunfw#65

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>

crates/krun/src/bin/krun.rs

@@ -11,10 +11,10 @@ use krun::launch::{launch_or_lock, LaunchResult};
 use krun::net::{connect_to_passt, start_passt};
 use krun::types::MiB;
 use krun_sys::{
- krun_add_vsock_port, krun_create_ctx, krun_set_exec, krun_set_gpu_options, krun_set_log_level,
- krun_set_passt_fd, krun_set_root, krun_set_vm_config, krun_set_workdir, krun_start_enter,
- VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC, VIRGLRENDERER_USE_ASYNC_FENCE_CB,
- VIRGLRENDERER_USE_EGL,
+ krun_add_disk, krun_add_vsock_port, krun_create_ctx, krun_set_exec, krun_set_gpu_options,
+ krun_set_log_level, krun_set_passt_fd, krun_set_root, krun_set_vm_config, krun_set_workdir,
+ krun_start_enter, VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC,
+ VIRGLRENDERER_USE_ASYNC_FENCE_CB, VIRGLRENDERER_USE_EGL,
 };
 use log::debug;
 use nix::sys::sysinfo::sysinfo;
@@ -24,6 +24,23 @@ use rustix::process::{
  geteuid, getgid, getrlimit, getuid, sched_setaffinity, setrlimit, CpuSet, Resource,
 };
 
+fn add_ro_disk(ctx_id: u32, label: &str, path: &str) -> Result<()> {
+ let path_cstr = CString::new(path).unwrap();
+ let path_ptr = path_cstr.as_ptr();
+
+ let label_cstr = CString::new(label).unwrap();
+ let label_ptr = label_cstr.as_ptr();
+
+ // SAFETY: `path_ptr` and `label_ptr` are live pointers to C-strings
+ let err = unsafe { krun_add_disk(ctx_id, label_ptr, path_ptr, true) };
+
+ if err < 0 {
+ Err(Errno::from_raw_os_error(-err).into())
+ } else {
+ Ok(())
+ }
+}
+
 fn main() -> Result<()> {
  env_logger::init();
 
@@ -122,6 +139,32 @@ fn main() -> Result<()> {
  setrlimit(Resource::Nofile, rlim).context("Failed to raise `RLIMIT_NOFILE`")?;
  }
 
+ // If the user specified a disk image, we want to load and fail if it's missing. If the user
+ // did not specify a disk image, we want to load the system images if installed but fail
+ // gracefully if missing. This follows the principle of least surprise.
+ //
+ // What we don't want is a clever autodiscovery mechanism that searches $HOME for images.
+ // That's liable to blow up in exciting ways. Instead we require images to be selected
+ // explicitly, either on the CLI or hardcoded here.
+ let disks: Vec<String> = if !options.fex_images.is_empty() {
+ options.fex_images
+ } else {
+ let default_disks = vec![
+ "/usr/share/fex-emu/RootFS/default.erofs",
+ "/usr/share/fex-emu/overlays/mesa.erofs",
+ ];
+
+ default_disks
+ .iter()
+ .map(|x| x.to_string())
+ .filter(|x| Path::new(x).exists())
+ .collect()
+ };
+
+ for path in disks {
+ add_ro_disk(ctx_id, &path, &path).context("Failed to configure disk")?;
+ }
+
  {
  // SAFETY: `root_path` is a pointer to a C-string literal.
  let err = unsafe { krun_set_root(ctx_id, c"/".as_ptr()) };

crates/krun/src/cli_options.rs

@@ -13,6 +13,7 @@ pub struct Options {
  pub mem: Option<MiB>,
  pub passt_socket: Option<PathBuf>,
  pub server_port: u32,
+ pub fex_images: Vec<String>,
  pub command: PathBuf,
  pub command_args: Vec<String>,
 }
@@ -66,6 +67,15 @@ pub fn options() -> OptionParser<Options> {
  )
  .argument("MEM")
  .optional();
+ let fex_images = long("fex-image")
+ .short('f')
+ .help(
+ "Adds an erofs file to be mounted as a FEX rootfs.
+ May be specified multiple times.
+ First the base image, then overlays in order.",
+ )
+ .argument::<String>("FEX_IMAGE")
+ .many();
  let passt_socket = long("passt-socket")
  .help("Instead of starting passt, connect to passt socket at PATH")
  .argument("PATH")
@@ -89,6 +99,7 @@ pub fn options() -> OptionParser<Options> {
  mem,
  passt_socket,
  server_port,
+ fex_images,
  // positionals
  command,
  command_args,

crates/krun/src/guest/mount.rs

@@ -1,22 +1,101 @@
-use std::fs::File;
+use std::ffi::CString;
+use std::fs::{read_dir, File};
+use std::io::Write;
 use std::os::fd::AsFd;
 use std::path::Path;
 
 use anyhow::{Context, Result};
-use rustix::fs::CWD;
+use rustix::fs::{mkdir, symlink, Mode, CWD};
 use rustix::mount::{
  mount2, mount_bind, move_mount, open_tree, MountFlags, MoveMountFlags, OpenTreeFlags,
 };
 
-pub fn mount_filesystems() -> Result<()> {
+fn make_tmpfs(dir: &str) -> Result<()> {
  mount2(
  Some("tmpfs"),
- "/var/run",
+ dir,
  Some("tmpfs"),
  MountFlags::NOEXEC | MountFlags::NOSUID | MountFlags::RELATIME,
  None,
  )
- .context("Failed to mount `/var/run`")?;
+ .context("Failed to mount tmpfs")
+}
+
+fn mkdir_fex(dir: &str) {
+ // Must succeed since /run/ was just mounted and is now an empty tmpfs.
+ mkdir(
+ dir,
+ Mode::RUSR | Mode::XUSR | Mode::RGRP | Mode::XGRP | Mode::ROTH | Mode::XOTH,
+ )
+ .unwrap();
+}
+
+fn mount_fex_rootfs() -> Result<()> {
+ let dir = "/run/fex-emu/";
+ let dir_rootfs = dir.to_string() + "rootfs";
+
+ // Make base directories
+ mkdir_fex(dir);
+
+ let flags = MountFlags::RDONLY;
+ let mut images = Vec::new();
+
+ // Find /dev/vd*
+ for x in read_dir("/dev").unwrap() {
+ let file = x.unwrap();
+ let name = file.file_name().into_string().unwrap();
+ if !name.starts_with("vd") {
+ continue;
+ }
+
+ let path = file.path().into_os_string().into_string().unwrap();
+ let dir = dir.to_string() + &name;
+
+ // Mount the erofs images.
+ mkdir_fex(&dir);
+ mount2(Some(path), dir.clone(), Some("erofs"), flags, None)
+ .context("Failed to mount erofs")
+ .unwrap();
+ images.push(dir);
+ }
+
+ if images.len() >= 2 {
+ // Overlay the mounts together.
+ let opts = format!(
+ "lowerdir={}",
+ images.into_iter().rev().collect::<Vec<String>>().join(":")
+ );
+ let opts = CString::new(opts).unwrap();
+ let overlay = "overlay".to_string();
+ let overlay_ = Some(&overlay);
+
+ mkdir_fex(&dir_rootfs);
+ mount2(overlay_, &dir_rootfs, overlay_, flags, Some(&opts)).context("Failed to overlay")?;
+ } else if images.len() == 1 {
+ // Just expose the one mount
+ symlink(&images[0], &dir_rootfs)?;
+ }
+
+ // Now we need to tell FEX about this. One of the FEX share directories has an unmounted rootfs
+ // and a Config.json telling FEX to use FUSE. Neither should be visible to the guest. Instead,
+ // we want to replace the folders and tell FEX to use our mounted rootfs
+ for base in ["/usr/share/fex-emu", "/usr/local/share/fex-emu"] {
+ let json = format!("{{\"Config\":{{\"RootFS\":\"{dir_rootfs}\"}}}}\n");
+ let path = base.to_string() + "/Config.json";
+
+ make_tmpfs(base)?;
+ File::create(Path::new(&path))?.write_all(json.as_bytes())?;
+ }
+
+ Ok(())
+}
+
+pub fn mount_filesystems() -> Result<()> {
+ make_tmpfs("/var/run")?;
+
+ if let Err(_) = mount_fex_rootfs() {
+ println!("Failed to mount FEX rootfs, carrying on without.")
+ }
 
  let _ = File::options()
  .write(true)
@@ -60,14 +139,7 @@ pub fn mount_filesystems() -> Result<()> {
  if Path::new("/tmp/.X11-unix").exists() {
  // Mount a tmpfs for X11 sockets, so the guest doesn't clobber host X server
  // sockets
- mount2(
- Some("tmpfs"),
- "/tmp/.X11-unix",
- Some("tmpfs"),
- MountFlags::NOEXEC | MountFlags::NOSUID | MountFlags::RELATIME,
- None,
- )
- .context("Failed to mount `/tmp/.X11-unix`")?;
+ make_tmpfs("/tmp/.X11-unix")?;
  }
 
  Ok(())