From 59daa4739cd4fece652b539b9acc4f3816944d64 Mon Sep 17 00:00:00 2001 From: anti-entropy123 <1348651580@qq.com> Date: Wed, 15 Jan 2025 15:48:04 +0800 Subject: [PATCH 1/4] prepare to test map reduce --- common_service/fatfs/src/apis.rs | 6 +-- isol_config/map_reduce_base.json | 79 ++++++++++++++++++++++++++++ isol_config/map_reduce_large_c1.json | 2 +- isol_config/map_reduce_large_c5.json | 6 ++- justfile | 32 +++++++---- ms_hostcall/src/fatfs.rs | 3 ++ ms_std/src/fs.rs | 3 ++ scripts/gen_data.py | 10 ++-- user/load_all/Cargo.lock | 66 ++++++++++++++++++++--- user/load_all/src/lib.rs | 3 -- 10 files changed, 181 insertions(+), 29 deletions(-) create mode 100644 isol_config/map_reduce_base.json diff --git a/common_service/fatfs/src/apis.rs b/common_service/fatfs/src/apis.rs index b709af9d..ba059fc4 100644 --- a/common_service/fatfs/src/apis.rs +++ b/common_service/fatfs/src/apis.rs @@ -59,7 +59,7 @@ pub fn fatfs_stat(fd: Fd) -> FatfsResult { let st_size = f .stream_len() .map_err(|e| FatfsError::HostIOErr(e.to_string()))? as Size; - Ok(Stat { + Ok(Stat { st_dev: 0, st_ino: 0, st_nlink: 0, @@ -83,7 +83,7 @@ pub fn fatfs_stat(fd: Fd) -> FatfsResult { tv_sec: 0, tv_nsec: 0, }, - __unused: [0, 0, 0], + __unused: [0, 0, 0], }) } @@ -100,7 +100,7 @@ pub fn fatfs_open(p: &str, flags: OpenFlags) -> FatfsResult { } else { root_dir.open_file(p) } - .map_err(|e| FatfsError::HostIOErr(e.to_string()))?; + .map_err(|_e| FatfsError::Unknown)?; let fd = { let file = ManuallyDrop::new(Box::new(file)); diff --git a/isol_config/map_reduce_base.json b/isol_config/map_reduce_base.json new file mode 100644 index 00000000..00ac32f0 --- /dev/null +++ b/isol_config/map_reduce_base.json @@ -0,0 +1,79 @@ +{ + "services": [ + [ + "fdtab", + "libfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "mmap_file_backend", + "libmmap_file_backend.so" + ], + [ + "socket", + "libsocket.so" + ], + [ + "time", + "libtime.so" + ], + [ + "fatfs", + "libfatfs.so" + ] + ], + "apps": [ + [ + "mapper", + "libmapper.so" + ], + [ + "reducer", + "libreducer.so" + ], + [ + "load_all", + "libload_all.so" + ] + ], + "fs_image": "fs_images/fatfs.img", + "groups": [ + { + "list": [ + "load_all" + ], + "args": {} + }, + { + "list": [ + "mapper", + "mapper", + "mapper", + "mapper", + "mapper" + ], + "args": { + "reducer_num": "5" + } + }, + { + "list": [ + "reducer", + "reducer", + "reducer", + "reducer", + "reducer" + ], + "args": { + "mapper_num": "5" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/map_reduce_large_c1.json b/isol_config/map_reduce_large_c1.json index 3f883530..f060f9fb 100644 --- a/isol_config/map_reduce_large_c1.json +++ b/isol_config/map_reduce_large_c1.json @@ -31,7 +31,7 @@ "libreducer.so" ] ], - "fs_image": "fs_images/new_image.img", + "fs_image": "fs_images/fatfs.img", "groups": [ { "list": [ diff --git a/isol_config/map_reduce_large_c5.json b/isol_config/map_reduce_large_c5.json index 5ea45aae..8ecd0563 100644 --- a/isol_config/map_reduce_large_c5.json +++ b/isol_config/map_reduce_large_c5.json @@ -22,6 +22,10 @@ ] ], "apps": [ + [ + "file_reader", + "libfile_reader.so" + ], [ "mapper", "libmapper.so" @@ -31,7 +35,7 @@ "libreducer.so" ] ], - "fs_image": "fs_images/new_image.img", + "fs_image": "fs_images/fatfs.img", "groups": [ { "list": [ diff --git a/justfile b/justfile index 94b39ce9..91845e2b 100644 --- a/justfile +++ b/justfile @@ -4,7 +4,7 @@ set positional-arguments enable_mpk := "0" enable_pkey_per_func := "0" -enable_release := "0" +enable_release := "1" cmd_flag := if enable_mpk == "1" { if enable_pkey_per_func == "1" { @@ -26,22 +26,32 @@ release_flag := if enable_release == "1" { "--release" } else { "" } +rust_func func_name: + cargo build {{ release_flag }} {{ feature_flag }} --manifest-path user/{{ func_name }}/Cargo.toml + +libos lib_name: + cargo build {{ release_flag }} {{ if enable_mpk == "1" { "--features mpk" } else { "" } }} --manifest-path common_service/{{ lib_name }}/Cargo.toml + pass_args: - cargo build --manifest-path user/func_a/Cargo.toml && \ - cargo build --manifest-path user/func_b/Cargo.toml + just rust_func func_a + just rust_func func_b map_reduce: - cargo build --manifest-path user/mapper/Cargo.toml && \ - cargo build --manifest-path user/reducer/Cargo.toml + for name in time fdtab fatfs stdio mm; do \ + just libos $name; \ + done + + just rust_func mapper + just rust_func reducer parallel_sort: - for func in file_reader sorter splitter merger; do \ - if ! cargo build --manifest-path user/$func/Cargo.toml; then \ - echo "build $func failed."; \ - exit 1; \ - fi; \ + for name in time fdtab fatfs stdio mm; do \ + just libos $name; \ done + for func in file_reader sorter splitter merger; do \ + just rust_func $func; \ + done all_libos: ./scripts/build_all_common{{ if enable_mpk == "1" { "_mpk" } else { "" } }}.sh {{ release_flag }} @@ -167,4 +177,4 @@ cpython_parallel_sort: cpython_parallel_sort_so all_py_wasm: cpython_wordcount cpython_parallel_sort -all_wasm: all_c_wasm all_py_wasm +all_wasm: all_c_wasm all_py_wasm \ No newline at end of file diff --git a/ms_hostcall/src/fatfs.rs b/ms_hostcall/src/fatfs.rs index e6ed27aa..7c588054 100644 --- a/ms_hostcall/src/fatfs.rs +++ b/ms_hostcall/src/fatfs.rs @@ -13,6 +13,7 @@ pub type FatfsStatFunc = fn(Fd) -> FatfsResult; pub type FatfsResult = Result; #[derive(Debug, Error)] +#[repr(C)] pub enum FatfsError { #[error("std::io::error, {0}")] HostIOErr(String), @@ -20,4 +21,6 @@ pub enum FatfsError { AcquireLockErr(String), #[error("bad input fd: {0}, do not exist")] BadInputFd(Fd), + #[error("Unknown.")] + Unknown, } diff --git a/ms_std/src/fs.rs b/ms_std/src/fs.rs index 264fb3b5..1392add6 100644 --- a/ms_std/src/fs.rs +++ b/ms_std/src/fs.rs @@ -6,6 +6,7 @@ use ms_hostcall::{ use crate::{ io::{Read, Write}, libos::libos, + println, }; pub struct File { @@ -24,7 +25,9 @@ impl File { pub fn open(p: &str) -> Result { let mode = OpenMode::RD; let flags = OpenFlags::empty(); + // println!("open file: {}", p); let raw_fd = libos!(open(p, flags, mode))?; + // println!("open file: {} ok", p); Ok(File { raw_fd }) } diff --git a/scripts/gen_data.py b/scripts/gen_data.py index a846e406..0e8026dd 100755 --- a/scripts/gen_data.py +++ b/scripts/gen_data.py @@ -4,6 +4,7 @@ workdir = "image_content" + def gen_word_count(file_num: int, total_size: int): from faker import Faker @@ -30,12 +31,15 @@ def gen_parallel_sort(file_num: int, total_size: int): current_size = 0 with open(file_name, 'w', encoding='utf-8') as f: while current_size < one_size: - text = ','.join(str(random.randint(0, 1000000)) for i in range(10)) + text = ','.join(str(random.randint(0, 1000000)) + for i in range(10)) text += ',' f.write(text) current_size += len(text.encode('utf-8')) # 更新当前文件大小 - + f.write('1') + if __name__ == "__main__": - gen_parallel_sort(3, 25 * 1024 * 1024) + # gen_parallel_sort(3, 25 * 1024 * 1024) + gen_word_count(5, 1 * 1024 * 1024) diff --git a/user/load_all/Cargo.lock b/user/load_all/Cargo.lock index c967dc95..6c56ed9c 100644 --- a/user/load_all/Cargo.lock +++ b/user/load_all/Cargo.lock @@ -64,6 +64,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + [[package]] name = "linked_list_allocator" version = "0.10.5" @@ -92,6 +98,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + [[package]] name = "ms_hostcall" version = "0.1.0" @@ -110,6 +122,8 @@ dependencies = [ "linked_list_allocator", "ms_hostcall", "ms_std_proc_macro", + "serde", + "serde_json", "spin", "thiserror-no-std", ] @@ -120,23 +134,23 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 1.0.109", ] [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -150,6 +164,12 @@ dependencies = [ "semver", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + [[package]] name = "scopeguard" version = "1.2.0" @@ -162,6 +182,38 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "serde_json" +version = "1.0.135" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + [[package]] name = "spin" version = "0.9.8" @@ -199,9 +251,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", diff --git a/user/load_all/src/lib.rs b/user/load_all/src/lib.rs index d61e349c..fcd73324 100644 --- a/user/load_all/src/lib.rs +++ b/user/load_all/src/lib.rs @@ -17,9 +17,6 @@ pub fn main() -> Result<()> { println!("Hello, world! id: {}", args::get("id").unwrap()); libos!(addrinfo("localhost")).unwrap(); - if let Err(e) = File::open(".") { - println!("{:#?}", e); - } Ok(().into()) } From 52025dabf062bbe26c38b545c43f5342d3009f59 Mon Sep 17 00:00:00 2001 From: anti-entropy123 <1348651580@qq.com> Date: Wed, 15 Jan 2025 15:59:15 +0800 Subject: [PATCH 2/4] add function feature to switch buffer impl --- justfile | 3 ++- user/file_reader/Cargo.toml | 1 + user/mapper/Cargo.toml | 2 ++ user/merger/Cargo.toml | 2 ++ user/reducer/Cargo.toml | 2 ++ user/sorter/Cargo.toml | 1 + user/splitter/Cargo.toml | 2 ++ 7 files changed, 12 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 91845e2b..28170f94 100644 --- a/justfile +++ b/justfile @@ -3,6 +3,7 @@ set positional-arguments enable_mpk := "0" enable_pkey_per_func := "0" +enable_file_buffer := "1" enable_release := "1" @@ -27,7 +28,7 @@ release_flag := if enable_release == "1" { } else { "" } rust_func func_name: - cargo build {{ release_flag }} {{ feature_flag }} --manifest-path user/{{ func_name }}/Cargo.toml + cargo build {{ release_flag }} {{ feature_flag }} {{ if enable_file_buffer == "1" { "--features file-based" } else { "" } }} --manifest-path user/{{ func_name }}/Cargo.toml libos lib_name: cargo build {{ release_flag }} {{ if enable_mpk == "1" { "--features mpk" } else { "" } }} --manifest-path common_service/{{ lib_name }}/Cargo.toml diff --git a/user/file_reader/Cargo.toml b/user/file_reader/Cargo.toml index e16930ba..75bfab26 100644 --- a/user/file_reader/Cargo.toml +++ b/user/file_reader/Cargo.toml @@ -18,5 +18,6 @@ serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] +file-based = ["ms_std/file-based"] default = [] diff --git a/user/mapper/Cargo.toml b/user/mapper/Cargo.toml index 74ab023b..f3d780c0 100644 --- a/user/mapper/Cargo.toml +++ b/user/mapper/Cargo.toml @@ -23,4 +23,6 @@ serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] +file-based = ["ms_std/file-based"] + default = [] diff --git a/user/merger/Cargo.toml b/user/merger/Cargo.toml index 4d3d21c9..b4b65c74 100644 --- a/user/merger/Cargo.toml +++ b/user/merger/Cargo.toml @@ -18,4 +18,6 @@ serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] +file-based = ["ms_std/file-based"] + default = [] diff --git a/user/reducer/Cargo.toml b/user/reducer/Cargo.toml index 3082e2e1..786cf1d4 100644 --- a/user/reducer/Cargo.toml +++ b/user/reducer/Cargo.toml @@ -23,4 +23,6 @@ serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] +file-based = ["ms_std/file-based"] + default = [] diff --git a/user/sorter/Cargo.toml b/user/sorter/Cargo.toml index 057bd3e7..cb7ab8a3 100644 --- a/user/sorter/Cargo.toml +++ b/user/sorter/Cargo.toml @@ -18,5 +18,6 @@ serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] +file-based = ["ms_std/file-based"] default = [] diff --git a/user/splitter/Cargo.toml b/user/splitter/Cargo.toml index bb2eb749..95ca6972 100644 --- a/user/splitter/Cargo.toml +++ b/user/splitter/Cargo.toml @@ -18,4 +18,6 @@ serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] +file-based = ["ms_std/file-based"] + default = [] From b488de43bd4d6f2a2af1708259398341a7ddbc44 Mon Sep 17 00:00:00 2001 From: anti-entropy123 <1348651580@qq.com> Date: Wed, 15 Jan 2025 19:26:46 +0800 Subject: [PATCH 3/4] finish --- isol_config/long_chain_load_all.json | 170 ++++++++++++++++++ isol_config/long_chain_n15.json | 90 ++++++---- ...uce_base.json => map_reduce_load_all.json} | 0 isol_config/parallel_sort.json | 5 +- isol_config/parallel_sort_load_all.json | 141 +++++++++++++++ justfile | 23 ++- scripts/build_user.sh | 1 - scripts/gen_data.py | 4 +- user/array_sum/Cargo.lock | 1 + user/array_sum/Cargo.toml | 5 +- user/array_sum/src/lib.rs | 63 +++---- 11 files changed, 426 insertions(+), 77 deletions(-) create mode 100644 isol_config/long_chain_load_all.json rename isol_config/{map_reduce_base.json => map_reduce_load_all.json} (100%) create mode 100644 isol_config/parallel_sort_load_all.json diff --git a/isol_config/long_chain_load_all.json b/isol_config/long_chain_load_all.json new file mode 100644 index 00000000..77698626 --- /dev/null +++ b/isol_config/long_chain_load_all.json @@ -0,0 +1,170 @@ +{ + "services": [ + [ + "mm", + "libmm.so" + ], + [ + "mmap_file_backend", + "libmmap_file_backend.so" + ], + [ + "socket", + "libsocket.so" + ], + [ + "time", + "libtime.so" + ], + [ + "fdtab", + "libfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "fatfs", + "libfatfs.so" + ] + ], + "apps": [ + [ + "array_sum", + "libarray_sum.so" + ], + [ + "load_all", + "libload_all.so" + ] + ], + "groups": [ + { + "list": [ + "load_all" + ], + "args": {} + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "0" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "1" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "2" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "3" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "4" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "5" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "6" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "7" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "8" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "9" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "10" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "11" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "12" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "13" + } + }, + { + "list": [ + "array_sum" + ], + "args": { + "n": "14" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/long_chain_n15.json b/isol_config/long_chain_n15.json index ecd83002..9e2e812d 100644 --- a/isol_config/long_chain_n15.json +++ b/isol_config/long_chain_n15.json @@ -86,93 +86,123 @@ "groups": [ { "list": [ - "array_sum0" + "array_sum" ], - "args": {"array_num": "0"} + "args": { + "n": "0" + } }, { "list": [ - "array_sum1" + "array_sum" ], - "args": {"array_num": "1"} + "args": { + "n": "1" + } }, { "list": [ - "array_sum2" + "array_sum" ], - "args": {"array_num": "2"} + "args": { + "n": "2" + } }, { "list": [ - "array_sum3" + "array_sum" ], - "args": {"array_num": "3"} + "args": { + "n": "3" + } }, { "list": [ - "array_sum4" + "array_sum" ], - "args": {"array_num": "4"} + "args": { + "n": "4" + } }, { "list": [ - "array_sum5" + "array_sum" ], - "args": {"array_num": "5"} + "args": { + "n": "5" + } }, { "list": [ - "array_sum6" + "array_sum" ], - "args": {"array_num": "6"} + "args": { + "n": "6" + } }, { "list": [ - "array_sum7" + "array_sum" ], - "args": {"array_num": "7"} + "args": { + "n": "7" + } }, { "list": [ - "array_sum8" + "array_sum" ], - "args": {"array_num": "8"} + "args": { + "n": "8" + } }, { "list": [ - "array_sum9" + "array_sum" ], - "args": {"array_num": "9"} + "args": { + "n": "9" + } }, { "list": [ - "array_sum10" + "array_sum" ], - "args": {"array_num": "10"} + "args": { + "n": "10" + } }, { "list": [ - "array_sum11" + "array_sum" ], - "args": {"array_num": "11"} + "args": { + "n": "11" + } }, { "list": [ - "array_sum12" + "array_sum" ], - "args": {"array_num": "12"} + "args": { + "n": "12" + } }, { "list": [ - "array_sum13" + "array_sum" ], - "args": {"array_num": "13"} + "args": { + "n": "13" + } }, { "list": [ - "array_sum14" + "array_sum" ], - "args": {"array_num": "14"} + "args": { + "n": "14" + } } ] } \ No newline at end of file diff --git a/isol_config/map_reduce_base.json b/isol_config/map_reduce_load_all.json similarity index 100% rename from isol_config/map_reduce_base.json rename to isol_config/map_reduce_load_all.json diff --git a/isol_config/parallel_sort.json b/isol_config/parallel_sort.json index 1210cc9f..f84f06f0 100644 --- a/isol_config/parallel_sort.json +++ b/isol_config/parallel_sort.json @@ -15,6 +15,10 @@ [ "fatfs", "libfatfs.so" + ], + [ + "time", + "libtime.so" ] ], "apps": [ @@ -27,7 +31,6 @@ "libsorter.so" ] ], - "fs_image": "fs_images/fatfs_large_sort.img", "groups": [ { "list": [ diff --git a/isol_config/parallel_sort_load_all.json b/isol_config/parallel_sort_load_all.json new file mode 100644 index 00000000..8c1f756e --- /dev/null +++ b/isol_config/parallel_sort_load_all.json @@ -0,0 +1,141 @@ +{ + "services": [ + [ + "fdtab", + "libfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "mmap_file_backend", + "libmmap_file_backend.so" + ], + [ + "socket", + "libsocket.so" + ], + [ + "time", + "libtime.so" + ], + [ + "fatfs", + "libfatfs.so" + ] + ], + "apps": [ + [ + "load_all", + "libload_all.so" + ], + [ + "file_reader", + "libfile_reader.so" + ], + [ + "sorter", + "libsorter.so" + ], + [ + "splitter", + "libsplitter.so" + ], + [ + "merger", + "libmerger.so" + ], + [ + "checker", + "libchecker.so" + ] + ], + "groups": [ + { + "list": [ + "load_all" + ], + "args": {} + }, + { + "list": [ + { + "name": "file_reader", + "args": { + "slot_name": "input-part-0", + "input_file": "sort_data_0.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-1", + "input_file": "sort_data_1.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-2", + "input_file": "sort_data_2.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-3", + "input_file": "sort_data_3.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-4", + "input_file": "sort_data_4.txt" + } + } + ], + "args": {} + }, + { + "list": [ + "sorter", + "sorter", + "sorter", + "sorter", + "sorter" + ], + "args": { + "sorter_num": "5", + "merger_num": "5" + } + }, + { + "list": [ + "splitter", + "splitter", + "splitter", + "splitter", + "splitter" + ], + "args": {} + }, + { + "list": [ + "merger", + "merger", + "merger", + "merger", + "merger" + ], + "args": { + "sorter_num": "5" + } + } + ] +} \ No newline at end of file diff --git a/justfile b/justfile index 28170f94..848fa74d 100644 --- a/justfile +++ b/justfile @@ -3,7 +3,7 @@ set positional-arguments enable_mpk := "0" enable_pkey_per_func := "0" -enable_file_buffer := "1" +enable_file_buffer := "0" enable_release := "1" @@ -54,6 +54,14 @@ parallel_sort: just rust_func $func; \ done +long_chain: + for name in time fdtab stdio mm; do \ + just libos $name; \ + done + + just rust_func array_sum + + all_libos: ./scripts/build_all_common{{ if enable_mpk == "1" { "_mpk" } else { "" } }}.sh {{ release_flag }} @@ -132,8 +140,6 @@ c_checker_so: {{cc_flags_p2}} \ -o target/{{target}}/debug/libwasmtime_checker.so - - wasmtime_parallel_sort: c_sorter_so c_spliter_so c_merger_so c_checker_so @echo "make symbol link: wasmtime_parallel_sort" @-rm target/debug/libwasmtime_sorter.so @@ -178,4 +184,13 @@ cpython_parallel_sort: cpython_parallel_sort_so all_py_wasm: cpython_wordcount cpython_parallel_sort -all_wasm: all_c_wasm all_py_wasm \ No newline at end of file +all_wasm: all_c_wasm all_py_wasm + +measure_avg isol_file: + #!/bin/bash + for (( i=1; i<=10; i++ )); do \ + output=$(cargo run {{ release_flag }} {{feature_flag}} -- --files {{ isol_file }} --metrics total-dur 2>&1); \ + total_dur=$(echo "$output" | grep -o '"total_dur(ms)": [0-9.]*' | awk -F': ' '{print $2}'); \ + total_dur_rounded=$(printf "%.3f\n" "$total_dur") ;\ + echo "$total_dur_rounded ," ;\ + done ; \ No newline at end of file diff --git a/scripts/build_user.sh b/scripts/build_user.sh index 307f875d..44f9bcce 100755 --- a/scripts/build_user.sh +++ b/scripts/build_user.sh @@ -22,7 +22,6 @@ for file in $(find user -name 'Cargo.toml' \ -not -path 'user/mmap_file/Cargo.toml' \ -not -path 'user/httpd/Cargo.toml' \ -not -path 'user/should_panic/Cargo.toml' \ - -not -path 'user/load_all/Cargo.toml' \ -not -path 'user/never_stop/Cargo.toml' \ -not -path 'user/tinywasm*/Cargo.toml' \ -not -path 'user/wasmtime*/Cargo.toml'); do diff --git a/scripts/gen_data.py b/scripts/gen_data.py index 0e8026dd..648db938 100755 --- a/scripts/gen_data.py +++ b/scripts/gen_data.py @@ -41,5 +41,5 @@ def gen_parallel_sort(file_num: int, total_size: int): if __name__ == "__main__": - # gen_parallel_sort(3, 25 * 1024 * 1024) - gen_word_count(5, 1 * 1024 * 1024) + gen_parallel_sort(5, 1 * 1024 * 1024) + # gen_word_count(5, 1 * 1024 * 1024) diff --git a/user/array_sum/Cargo.lock b/user/array_sum/Cargo.lock index 233d39e5..e0a1cf20 100644 --- a/user/array_sum/Cargo.lock +++ b/user/array_sum/Cargo.lock @@ -9,6 +9,7 @@ dependencies = [ "ms_hostcall", "ms_std", "ms_std_proc_macro", + "serde", ] [[package]] diff --git a/user/array_sum/Cargo.toml b/user/array_sum/Cargo.toml index f5bb47b9..bb96f25d 100644 --- a/user/array_sum/Cargo.toml +++ b/user/array_sum/Cargo.toml @@ -12,8 +12,11 @@ crate-type = ["dylib"] ms_std = { path = "../../ms_std", features = ["alloc_def", "panic_def"] } ms_hostcall = { path = "../../ms_hostcall" } ms_std_proc_macro = { path = "../../ms_std_proc_macro" } +serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] -default = [] \ No newline at end of file +file-based = ["ms_std/file-based"] + +default = [] diff --git a/user/array_sum/src/lib.rs b/user/array_sum/src/lib.rs index 6bb1398f..8d2dde5e 100644 --- a/user/array_sum/src/lib.rs +++ b/user/array_sum/src/lib.rs @@ -9,53 +9,40 @@ use alloc::{format, string::String}; // println, // time::{SystemTime, UNIX_EPOCH}, // }; -use ms_std::agent::{DataBuffer, FaaSFuncResult as Result}; +use ms_std::{ + agent::{DataBuffer, FaaSFuncResult as Result}, + args, println, +}; use ms_std_proc_macro::FaasData; +use serde::{Deserialize, Serialize}; -#[derive(Default, FaasData)] -struct Reader2Arraysum { +#[derive(Default, FaasData, Serialize, Deserialize)] +struct Arraysum { raw_data: String, + count: i32, } +#[allow(clippy::identity_op)] const DATA_SIZE: usize = 1024 * 1024 * 1; #[allow(clippy::result_unit_err)] #[no_mangle] pub fn main() -> Result<()> { - // let my_id = args::get("id").unwrap(); - // let array_num: u64 = args::get("array_num") - // .expect("missing arg reducer_num") - // .parse() - // .unwrap_or_else(|_| panic!("bad arg, array_num={}", args::get("array_num").unwrap())); - // println!("get id {}", my_id); - // if array_num == 0 { - // let mut buffer: DataBuffer = DataBuffer::with_slot(format!("part-{}-{}", array_num, my_id)); - // buffer.raw_data = "a".repeat(DATA_SIZE); - // } else if array_num == 9 { - // let reader: DataBuffer = DataBuffer::from_buffer_slot(format!("part-{}-{}", array_num-1, my_id)).expect("missing input data."); - // let result = reader.raw_data.clone(); - // } else { - // let mut buffer: DataBuffer = DataBuffer::with_slot(format!("part-{}-{}", array_num, my_id)); - // let reader: DataBuffer = DataBuffer::from_buffer_slot(format!("part-{}-{}", array_num-1, my_id)).expect("missing input data."); - // buffer.raw_data = reader.raw_data.clone(); - // } - let mut buffer: DataBuffer = DataBuffer::with_slot(format!("part-{}", 0)); - buffer.raw_data = "a".repeat(DATA_SIZE); - // println!("hhh1"); - let mut count = 0; - while count < 13 { - // println!("hhh{}", count); - let mut buffer2: DataBuffer = - DataBuffer::with_slot(format!("part-{}", count + 1)); - let reader: DataBuffer = - DataBuffer::from_buffer_slot(format!("part-{}", count)).expect("missing input data."); - buffer2.raw_data = reader.raw_data.clone(); - count += 1; - } - // println!("hhh100"); - let reader2: DataBuffer = - DataBuffer::from_buffer_slot(format!("part-{}", 13)).expect("missing input data."); - let _result = reader2.raw_data.clone(); - // println!("hhh101"); + let n = args::get("n").expect("missing arg 'n'?"); + let i: i32 = n.parse().expect("wrong arg 'n' format"); + + let previous_cnt: i32 = if i == 0 { + 0 + } else { + DataBuffer::::from_buffer_slot(format!("slot_{}", i - 1)) + .expect("missing data buffer?") + .count + }; + + let mut next_buffer: DataBuffer = DataBuffer::with_slot(format!("slot_{}", n)); + next_buffer.raw_data = n.repeat(DATA_SIZE); + next_buffer.count = previous_cnt + 1; + println!("count is {}", next_buffer.count); + Ok(().into()) } From 4b39405c5ce2966f83b911589d0e35a61afaa864 Mon Sep 17 00:00:00 2001 From: anti-entropy123 <1348651580@qq.com> Date: Thu, 16 Jan 2025 12:51:05 +0800 Subject: [PATCH 4/4] fix ci failed. --- isol_config/long_chain_n15.json | 58 +-------------------------------- scripts/run_tests.sh | 2 +- 2 files changed, 2 insertions(+), 58 deletions(-) diff --git a/isol_config/long_chain_n15.json b/isol_config/long_chain_n15.json index 9e2e812d..c9d5efd0 100644 --- a/isol_config/long_chain_n15.json +++ b/isol_config/long_chain_n15.json @@ -23,63 +23,7 @@ ], "apps": [ [ - "array_sum0", - "libarray_sum.so" - ], - [ - "array_sum1", - "libarray_sum.so" - ], - [ - "array_sum2", - "libarray_sum.so" - ], - [ - "array_sum3", - "libarray_sum.so" - ], - [ - "array_sum4", - "libarray_sum.so" - ], - [ - "array_sum5", - "libarray_sum.so" - ], - [ - "array_sum6", - "libarray_sum.so" - ], - [ - "array_sum7", - "libarray_sum.so" - ], - [ - "array_sum8", - "libarray_sum.so" - ], - [ - "array_sum9", - "libarray_sum.so" - ], - [ - "array_sum10", - "libarray_sum.so" - ], - [ - "array_sum11", - "libarray_sum.so" - ], - [ - "array_sum12", - "libarray_sum.so" - ], - [ - "array_sum13", - "libarray_sum.so" - ], - [ - "array_sum14", + "array_sum", "libarray_sum.so" ] ], diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index f4934ca2..82586a26 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash declare -A test_groups=( - [faasflow_test]="long_chain map_reduce pass_complex_args" + [faasflow_test]="long_chain_n15 pass_complex_args map_reduce parallel_sort" [file_test]="simple_file" # [wasm_test]="tinywasm_wordcount tinywasm_parallel_sort tinywasm_cpython" )