Merge #608

608: Implement more wasi syscalls r=MarkMcCaskey a=MarkMcCaskey



Co-authored-by: Mark McCaskey <mark@wasmer.io>
Co-authored-by: Mark McCaskey <markmccaskey@users.noreply.github.com>

CHANGELOG.md

@@ -5,8 +5,10 @@ All PRs to the Wasmer repository must add to this file.
 Blocks of changes will separated by version increments.
 
 ## **[Unreleased]**
+
 Special thanks to @YaronWittenstein @penberg for their contributions.
 
+- [#608](https://github.com/wasmerio/wasmer/issues/608) Implement wasi syscalls `fd_allocate`, `fd_sync`, `fd_pread`, `path_link`, `path_filestat_set_times`; update WASI fs API in a WIP way; reduce coupling of WASI code to host filesystem; make debug messages from WASI more readable; improve rights-checking when calling syscalls; implement reference counting on inodes; misc bug fixes and improvements
 - [#616](https://github.com/wasmerio/wasmer/issues/616) Create the import object separately from instance instantiation in `runtime-c-api`
 - [#620](https://github.com/wasmerio/wasmer/issues/620) Replace one `throw()` with `noexcept` in llvm backend
 - [#618](https://github.com/wasmerio/wasmer/issues/618) Implement `InternalEvent::Breakpoint` in the llvm backend to allow metering in llvm

Makefile

@@ -62,13 +62,17 @@ middleware: middleware-singlepass middleware-cranelift middleware-llvm
 
 
 # Wasitests
-wasitests-singlepass:
+wasitests-setup:
+	rm -rf lib/wasi-tests/wasitests/test_fs/temp
+	mkdir -p lib/wasi-tests/wasitests/test_fs/temp
+
+wasitests-singlepass: wasitests-setup
 	cargo test --manifest-path lib/wasi-tests/Cargo.toml --release --features singlepass -- --test-threads=1
 
-wasitests-cranelift:
+wasitests-cranelift: wasitests-setup
 	cargo test --manifest-path lib/wasi-tests/Cargo.toml --release --features clif -- --test-threads=1
 
-wasitests-llvm:
+wasitests-llvm: wasitests-setup
 	cargo test --manifest-path lib/wasi-tests/Cargo.toml --release --features llvm -- --test-threads=1
 
 wasitests-unit:

lib/wasi-tests/build/wasitests.rs

@@ -63,6 +63,22 @@ pub fn compile(file: &str, ignores: &HashSet<String>) -> Option<String> {
     std::fs::remove_file(&normalized_name).expect("could not delete executable");
     let wasm_out_name = format!("{}.wasm", &normalized_name);
 
+    let mut file_contents = String::new();
+    {
+        let mut file = std::fs::File::open(file).unwrap();
+        file.read_to_string(&mut file_contents).unwrap();
+    }
+    {
+        let mut actual_file = std::fs::OpenOptions::new()
+            .write(true)
+            .truncate(true)
+            .open(file)
+            .unwrap();
+        actual_file
+            .write_all(format!("#![feature(wasi_ext)]\n{}", &file_contents).as_bytes())
+            .unwrap();
+    }
+
     let wasm_compilation_out = Command::new("rustc")
         .arg("+nightly")
         .arg("--target=wasm32-wasi")
@@ -74,6 +90,14 @@ pub fn compile(file: &str, ignores: &HashSet<String>) -> Option<String> {
         .output()
         .expect("Failed to compile program to native code");
     print_info_on_error(&wasm_compilation_out, "WASM COMPILATION");
+    {
+        let mut actual_file = std::fs::OpenOptions::new()
+            .write(true)
+            .truncate(true)
+            .open(file)
+            .unwrap();
+        actual_file.write_all(file_contents.as_bytes()).unwrap();
+    }
 
     // to prevent commiting huge binary blobs forever
     let wasm_strip_out = Command::new("wasm-strip")

lib/wasi-tests/tests/wasitests/fd_allocate.rs

@@ -0,0 +1,14 @@
+#[test]
+fn test_fd_allocate() {
+    assert_wasi_output!(
+        "../../wasitests/fd_allocate.wasm",
+        "fd_allocate",
+        vec![],
+        vec![(
+            ".".to_string(),
+            ::std::path::PathBuf::from("wasitests/test_fs/temp")
+        ),],
+        vec![],
+        "../../wasitests/fd_allocate.out"
+    );
+}

lib/wasi-tests/tests/wasitests/fd_pread.rs

@@ -0,0 +1,14 @@
+#[test]
+fn test_fd_pread() {
+    assert_wasi_output!(
+        "../../wasitests/fd_pread.wasm",
+        "fd_pread",
+        vec![],
+        vec![(
+            ".".to_string(),
+            ::std::path::PathBuf::from("wasitests/test_fs/hamlet")
+        ),],
+        vec![],
+        "../../wasitests/fd_pread.out"
+    );
+}

lib/wasi-tests/tests/wasitests/fd_sync.rs

@@ -0,0 +1,14 @@
+#[test]
+fn test_fd_sync() {
+    assert_wasi_output!(
+        "../../wasitests/fd_sync.wasm",
+        "fd_sync",
+        vec![],
+        vec![(
+            ".".to_string(),
+            ::std::path::PathBuf::from("wasitests/test_fs/temp")
+        ),],
+        vec![],
+        "../../wasitests/fd_sync.out"
+    );
+}

lib/wasi-tests/tests/wasitests/mod.rs

@@ -7,11 +7,15 @@
 mod _common;
 mod create_dir;
 mod envvar;
+mod fd_allocate;
+mod fd_pread;
+mod fd_sync;
 mod file_metadata;
 mod fs_sandbox_test;
 mod fseek;
 mod hello;
 mod mapdir;
+mod path_link;
 mod quine;
 mod readlink;
 mod wasi_sees_virtual_root;

lib/wasi-tests/tests/wasitests/path_link.rs

@@ -0,0 +1,20 @@
+#[test]
+fn test_path_link() {
+    assert_wasi_output!(
+        "../../wasitests/path_link.wasm",
+        "path_link",
+        vec![],
+        vec![
+            (
+                "act5".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/hamlet/act5")
+            ),
+            (
+                "temp".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/temp")
+            ),
+        ],
+        vec![],
+        "../../wasitests/path_link.out"
+    );
+}

lib/wasi-tests/wasitests/fd_allocate.out

@@ -0,0 +1,2 @@
+171
+1405

lib/wasi-tests/wasitests/fd_allocate.rs

@@ -0,0 +1,58 @@
+// Args:
+// mapdir: .:wasitests/test_fs/temp
+
+use std::fs;
+#[cfg(target_os = "wasi")]
+use std::os::wasi::prelude::AsRawFd;
+use std::path::PathBuf;
+
+#[cfg(target_os = "wasi")]
+#[link(wasm_import_module = "wasi_unstable")]
+extern "C" {
+    fn fd_allocate(fd: u32, offset: u64, length: u64) -> u16;
+}
+
+#[cfg(target_os = "wasi")]
+fn allocate(fd: u32, offset: u64, length: u64) -> u16 {
+    unsafe { fd_allocate(fd, offset, length) }
+}
+
+fn main() {
+    #[cfg(target_os = "wasi")]
+    let mut base = PathBuf::from(".");
+    #[cfg(target_os = "wasi")]
+    {
+        base.push("fd_allocate_file.txt");
+        let mut file = fs::OpenOptions::new()
+            .create(true)
+            .write(true)
+            .truncate(true)
+            .open(&base)
+            .expect("Could not create file");
+        let mut buffer = [0u8; 64];
+
+        {
+            use std::io::Write;
+            // example text from https://www.un.org/en/universal-declaration-human-rights/
+            file.write_all(b"All human beings are born free and equal in dignity and rights. They are endowed with reason and conscience and should act towards one another in a spirit of brotherhood.\n").unwrap();
+            let raw_fd = file.as_raw_fd();
+            file.flush();
+            let len = file.metadata().unwrap().len();
+            println!("{}", len);
+            assert_eq!(len, 171);
+            allocate(raw_fd, len, 1234);
+            let len = file.metadata().unwrap().len();
+            println!("{}", len);
+            assert_eq!(len, 1234 + 171);
+        }
+    }
+    #[cfg(target_os = "wasi")]
+    std::fs::remove_file(&base).unwrap();
+
+    #[cfg(not(target_os = "wasi"))]
+    {
+        // eh, just print the output directly
+        println!("171");
+        println!("1405");
+    }
+}

lib/wasi-tests/wasitests/fd_pread.out

@@ -0,0 +1,9 @@
+ POLONIUS
+
+    He will come straight. Look you lay home to him:
+
+ POLONIUS
+
+    He will come straight. Look you lay home to him:
+
+Read the same data? true

lib/wasi-tests/wasitests/fd_pread.rs

@@ -0,0 +1,87 @@
+// Args:
+// mapdir: .:wasitests/test_fs/hamlet
+
+use std::fs;
+#[cfg(target_os = "wasi")]
+use std::os::wasi::prelude::AsRawFd;
+use std::path::PathBuf;
+
+#[cfg(target_os = "wasi")]
+#[repr(C)]
+struct WasiIovec {
+    pub buf: u32,
+    pub buf_len: u32,
+}
+
+#[cfg(target_os = "wasi")]
+#[link(wasm_import_module = "wasi_unstable")]
+extern "C" {
+    fn fd_pread(fd: u32, iovs: u32, iovs_len: u32, offset: u64, nread: u32) -> u16;
+}
+
+#[cfg(target_os = "wasi")]
+fn pread(fd: u32, iovs: &[&mut [u8]], offset: u64) -> u32 {
+    let mut nread = 0;
+    let mut processed_iovs = vec![];
+
+    for iov in iovs {
+        processed_iovs.push(WasiIovec {
+            buf: iov.as_ptr() as usize as u32,
+            buf_len: iov.len() as u32,
+        })
+    }
+
+    unsafe {
+        fd_pread(
+            fd,
+            processed_iovs.as_ptr() as usize as u32,
+            processed_iovs.len() as u32,
+            offset,
+            &mut nread as *mut u32 as usize as u32,
+        );
+    }
+    nread
+}
+
+fn main() {
+    #[cfg(not(target_os = "wasi"))]
+    let mut base = PathBuf::from("wasitests/test_fs/hamlet");
+    #[cfg(target_os = "wasi")]
+    let mut base = PathBuf::from(".");
+
+    base.push("act3/scene4.txt");
+    let mut file = fs::File::open(&base).expect("Could not open file");
+    let mut buffer = [0u8; 64];
+
+    #[cfg(target_os = "wasi")]
+    {
+        let raw_fd = file.as_raw_fd();
+        assert_eq!(pread(raw_fd, &[&mut buffer], 75), 64);
+        let str_val = std::str::from_utf8(&buffer[..]).unwrap().to_string();
+        println!("{}", &str_val);
+        for i in 0..buffer.len() {
+            buffer[i] = 0;
+        }
+        assert_eq!(pread(raw_fd, &[&mut buffer], 75), 64);
+        let str_val2 = std::str::from_utf8(&buffer[..]).unwrap().to_string();
+        println!("{}", &str_val2);
+
+        println!("Read the same data? {}", str_val == str_val2);
+    }
+
+    #[cfg(not(target_os = "wasi"))]
+    {
+        // eh, just print the output directly
+        print!(
+            " POLONIUS
+
+    He will come straight. Look you lay home to him:
+
+ POLONIUS
+
+    He will come straight. Look you lay home to him:
+
+Read the same data? true"
+        );
+    }
+}

lib/wasi-tests/wasitests/fd_sync.out

@@ -0,0 +1,2 @@
+170
+1404

lib/wasi-tests/wasitests/fd_sync.rs

@@ -0,0 +1,56 @@
+// Args:
+// mapdir: .:wasitests/test_fs/temp
+
+use std::fs;
+use std::path::PathBuf;
+
+#[cfg(target_os = "wasi")]
+#[link(wasm_import_module = "wasi_unstable")]
+extern "C" {
+    fn fd_sync(fd: u32) -> u16;
+}
+
+#[cfg(target_os = "wasi")]
+fn sync(fd: u32) -> u16 {
+    unsafe { fd_sync(fd) }
+}
+
+fn main() {
+    #[cfg(target_os = "wasi")]
+    let mut base = PathBuf::from(".");
+    #[cfg(target_os = "wasi")]
+    {
+        base.push("fd_sync_file.txt");
+        let mut file = fs::OpenOptions::new()
+            .create(true)
+            .write(true)
+            .truncate(true)
+            .open(&base)
+            .expect("Could not create file");
+        let mut buffer = [0u8; 64];
+
+        {
+            use std::io::Write;
+            // example text from https://www.un.org/en/universal-declaration-human-rights/
+            file.write_all(b"All human beings are born free and equal in dignity and rights. They are endowed with reason and conscience and should act towards one another in a spirit of brotherhood.").unwrap();
+            file.sync_all();
+            let len = file.metadata().unwrap().len();
+            println!("{}", len);
+            assert_eq!(len, 170);
+            file.set_len(170 + 1234);
+            file.sync_all();
+            let len = file.metadata().unwrap().len();
+            println!("{}", len);
+            assert_eq!(len, 1234 + 170);
+        }
+    }
+    #[cfg(target_os = "wasi")]
+    std::fs::remove_file(&base).unwrap();
+
+    #[cfg(not(target_os = "wasi"))]
+    {
+        // eh, just print the output directly
+        println!("170");
+        println!("1404");
+    }
+}

lib/wasi-tests/wasitests/path_link.out

@@ -0,0 +1,9 @@
+ACT V
+SCENE I. A churchyard.
+
+    Enter two Clowns, with spades,
+ACT V
+SCENE I. A churchyard.
+
+    Enter two Clowns, with spades,
+Path still exists