diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 4d461d6a..5934692c 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -404,7 +404,7 @@ jobs: .pixi - name: Build vegafusion-server run: | - pixi run build-rs-vegafusion-server + pixi run build-rs-server - name: zip executable uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 with: @@ -441,7 +441,7 @@ jobs: # pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu # pixi add gcc_linux-aarch64 -p linux-64 # export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc" -# pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu +# pixi run build-rs-server --features=protobuf-src --target aarch64-unknown-linux-gnu # - name: zip executable # uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 # with: @@ -514,7 +514,7 @@ jobs: - name: Build vegafusion-server run: | pixi install -vvv - pixi run build-rs-vegafusion-server + pixi run build-rs-server - name: zip executable uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 with: @@ -545,7 +545,7 @@ jobs: .pixi - name: Build vegafusion-server run: | - pixi run build-rs-vegafusion-server + pixi run build-rs-server - name: zip executable uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 with: @@ -576,7 +576,7 @@ jobs: .pixi - name: Build vegafusion-server run: | - pixi run build-rs-vegafusion-server --target aarch64-apple-darwin + pixi run build-rs-server --target aarch64-apple-darwin - name: zip executable uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1 with: diff --git a/Cargo.lock b/Cargo.lock index 19960239..39c9668e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,9 +399,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.17" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "bzip2", "flate2", @@ -441,7 +441,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -463,7 +463,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -474,7 +474,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -563,6 +563,18 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backon" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" +dependencies = [ + "fastrand", + "futures-core", + "pin-project", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -867,10 +879,10 @@ version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -1102,7 +1114,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.10", + "parking_lot_core", ] [[package]] @@ -1146,7 +1158,7 @@ dependencies = [ "log", "num_cpus", "object_store", - "parking_lot 0.12.3", + "parking_lot", "parquet", "paste", "pin-project-lite", @@ -1172,7 +1184,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", - "parking_lot 0.12.3", + "parking_lot", ] [[package]] @@ -1222,7 +1234,7 @@ dependencies = [ "hashbrown 0.14.5", "log", "object_store", - "parking_lot 0.12.3", + "parking_lot", "rand", "tempfile", "url", @@ -1468,7 +1480,7 @@ dependencies = [ "itertools 0.13.0", "log", "once_cell", - "parking_lot 0.12.3", + "parking_lot", "pin-project-lite", "rand", "tokio", @@ -1563,6 +1575,15 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -1710,7 +1731,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -1865,6 +1886,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -2691,6 +2718,32 @@ dependencies = [ "memchr", ] +[[package]] +name = "object-store-wasm" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04a7032dd2b5fd910de2b9d72cb4fa250c899bd1586ecb3b4b334e75cd7868f7" +dependencies = [ + "async-trait", + "backon", + "bytes", + "chrono", + "console_error_panic_hook", + "futures", + "js-sys", + "object_store", + "reqwest 0.11.27", + "serde", + "serde-wasm-bindgen", + "snafu 0.7.5", + "tokio", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "object_store" version = "0.11.1" @@ -2706,15 +2759,15 @@ dependencies = [ "hyper 1.5.0", "itertools 0.13.0", "md-5", - "parking_lot 0.12.3", + "parking_lot", "percent-encoding", "quick-xml", "rand", - "reqwest", + "reqwest 0.12.8", "ring", "serde", "serde_json", - "snafu", + "snafu 0.8.5", "tokio", "tracing", "url", @@ -2763,17 +2816,6 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.3" @@ -2781,21 +2823,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core 0.9.10", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -2806,7 +2834,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.7", + "redox_syscall", "smallvec", "windows-targets 0.52.6", ] @@ -2934,7 +2962,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3065,7 +3093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.85", ] [[package]] @@ -3104,7 +3132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -3114,7 +3142,7 @@ dependencies = [ "prost 0.12.6", "prost-types", "regex", - "syn", + "syn 2.0.85", "tempfile", ] @@ -3128,7 +3156,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3141,7 +3169,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3228,7 +3256,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3237,11 +3265,11 @@ version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1e3f09eecd94618f60a455a23def79f79eba4dc561a97324bf9ac8c6df30ce" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3377,15 +3405,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.7" @@ -3430,6 +3449,44 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.31", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + [[package]] name = "reqwest" version = "0.12.8" @@ -3477,51 +3534,6 @@ dependencies = [ "windows-registry", ] -[[package]] -name = "reqwest-middleware" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" -dependencies = [ - "anyhow", - "async-trait", - "http 1.1.0", - "reqwest", - "serde", - "thiserror", - "tower-service", -] - -[[package]] -name = "reqwest-retry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a83df1aaec00176d0fabb65dea13f832d2a446ca99107afc17c5d2d4981221d0" -dependencies = [ - "anyhow", - "async-trait", - "futures", - "getrandom", - "http 1.1.0", - "hyper 1.5.0", - "parking_lot 0.11.2", - "reqwest", - "reqwest-middleware", - "retry-policies", - "tokio", - "tracing", - "wasm-timer", -] - -[[package]] -name = "retry-policies" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" -dependencies = [ - "rand", -] - [[package]] name = "rgb" version = "0.8.50" @@ -3571,7 +3583,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn", + "syn 2.0.85", "unicode-ident", ] @@ -3820,7 +3832,7 @@ checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3886,13 +3898,35 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive 0.7.5", +] + [[package]] name = "snafu" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" dependencies = [ - "snafu-derive", + "snafu-derive 0.8.5", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -3901,10 +3935,10 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3947,7 +3981,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -3977,11 +4011,11 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.85", ] [[package]] @@ -3990,6 +4024,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.85" @@ -4030,6 +4075,27 @@ dependencies = [ "windows", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -4082,7 +4148,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -4093,7 +4159,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", "test-case-core", ] @@ -4120,7 +4186,7 @@ checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -4213,7 +4279,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -4301,7 +4367,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -4393,7 +4459,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -4598,6 +4664,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "bytes", + "cfg-if", "chrono", "chrono-tz 0.9.0", "criterion", @@ -4621,14 +4688,13 @@ dependencies = [ "log", "lru", "num-traits", + "object-store-wasm", "object_store", "ordered-float 3.9.2", "pixelmatch", "prost 0.12.6", "regex", - "reqwest", - "reqwest-middleware", - "reqwest-retry", + "reqwest 0.12.8", "rgb", "rstest", "serde", @@ -4684,6 +4750,7 @@ dependencies = [ "serde_json", "vegafusion-common", "vegafusion-core", + "vegafusion-runtime", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", @@ -4752,7 +4819,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.85", "wasm-bindgen-shared", ] @@ -4786,7 +4853,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4820,7 +4887,7 @@ checksum = "c97b2ef2c8d627381e51c071c2ab328eac606d3f69dd82bcbca20a9e389d95f0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -4836,21 +4903,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasm-timer" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" -dependencies = [ - "futures", - "js-sys", - "parking_lot 0.11.2", - "pin-utils", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "web-sys" version = "0.3.72" @@ -4946,7 +4998,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -4957,7 +5009,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -5008,6 +5060,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5041,6 +5102,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -5063,6 +5139,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5075,6 +5157,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5087,6 +5175,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5105,6 +5199,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -5117,6 +5217,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -5129,6 +5235,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -5141,12 +5253,28 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xz2" version = "0.1.7" @@ -5174,7 +5302,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.85", ] [[package]] @@ -5185,27 +5313,27 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" -version = "0.13.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 085eb1a7..a8c4e9ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,38 +1,79 @@ [workspace] resolver = "2" -members = [ - "vegafusion-common", - "vegafusion-core", - "vegafusion-runtime", - "vegafusion-python", - "vegafusion-wasm", - "vegafusion-server", -] +members = [ "vegafusion-common", "vegafusion-core", "vegafusion-runtime", "vegafusion-python", "vegafusion-wasm", "vegafusion-server",] [workspace.dependencies] -arrow = { version = "53.1.0", default-features = false } - -sqlparser = { version = "0.51.0" } -chrono = { version = "0.4.35", default-features = false } -chrono-tz = { version = "0.9.0", features = [ - "case-insensitive", - "filter-by-regex", -] } deterministic-hash = "1.0.1" -reqwest = { version = "0.12.8", default-features = false } -reqwest-middleware = { version = "0.3" } -reqwest-retry = "0.6" -tokio = { version = "1.36.0" } -pyo3 = { version = "0.22.4" } -pythonize = { version = "0.22" } -pyo3-arrow = { version = "0.5.1", default-features = false } -prost = { version = "0.12.3" } -prost-types = { version = "0.12.3" } -object_store = { version = "0.11.0" } -lazy_static = { version = "1.5" } async-trait = "0.1.73" futures = "0.3.21" url = "2.3.1" +cfg-if = "1.0.0" + +[profile.release] +opt-level = 3 + +[profile.release-dev] +inherits = "release" +opt-level = 3 + +[profile.release-opt] +inherits = "release" +opt-level = 3 +codegen-units = 1 +lto = "fat" +strip = "debuginfo" + +[profile.release-small] +inherits = "release" +opt-level = "z" +codegen-units = 1 + +[workspace.dependencies.arrow] +version = "53.1.0" +default-features = false + +[workspace.dependencies.sqlparser] +version = "0.51.0" + +[workspace.dependencies.chrono] +version = "0.4.35" +default-features = false + +[workspace.dependencies.chrono-tz] +version = "0.9.0" +features = [ "case-insensitive", "filter-by-regex",] + +[workspace.dependencies.reqwest] +version = "0.12.8" +default-features = false + +[workspace.dependencies.tokio] +version = "1.36.0" + +[workspace.dependencies.pyo3] +version = "0.22.4" + +[workspace.dependencies.pythonize] +version = "0.22" + +[workspace.dependencies.pyo3-arrow] +version = "0.5.1" +default-features = false + +[workspace.dependencies.prost] +version = "0.12.3" + +[workspace.dependencies.prost-types] +version = "0.12.3" + +[workspace.dependencies.object_store] +version = "0.11.0" + +[workspace.dependencies.object-store-wasm] +version = "0.0.6" + +[workspace.dependencies.lazy_static] +version = "1.5" [workspace.dependencies.serde_json] version = "1.0.91" @@ -42,23 +83,22 @@ default-features = false git = "https://github.com/apache/datafusion.git" rev = "b30d12a73fb9867180c2fdf8ddc818b45f957bac" default-features = false -features = ["parquet", "nested_expressions"] +features = [ "nested_expressions",] [workspace.dependencies.datafusion-common] git = "https://github.com/apache/datafusion.git" rev = "b30d12a73fb9867180c2fdf8ddc818b45f957bac" -# no default features +default-features = false [workspace.dependencies.datafusion-expr] git = "https://github.com/apache/datafusion.git" rev = "b30d12a73fb9867180c2fdf8ddc818b45f957bac" -# no default features [workspace.dependencies.datafusion-proto] git = "https://github.com/apache/datafusion.git" rev = "b30d12a73fb9867180c2fdf8ddc818b45f957bac" default-features = false -features = ["parquet"] +features = [ "parquet",] [workspace.dependencies.datafusion-proto-common] git = "https://github.com/apache/datafusion.git" @@ -83,35 +123,11 @@ default-features = false [workspace.dependencies.datafusion-functions-nested] git = "https://github.com/apache/datafusion.git" rev = "b30d12a73fb9867180c2fdf8ddc818b45f957bac" -# no default features [workspace.dependencies.datafusion-functions-aggregate] git = "https://github.com/apache/datafusion.git" rev = "b30d12a73fb9867180c2fdf8ddc818b45f957bac" -# no default features [workspace.dependencies.datafusion-functions-window] git = "https://github.com/apache/datafusion.git" rev = "b30d12a73fb9867180c2fdf8ddc818b45f957bac" -# no default features - -# Profile with good speed for local development and testing -[profile.release] -opt-level = 3 - -# Profile optimized for speed and small size, -# but takes a long time to compile -[profile.release-opt] -inherits = "release" -opt-level = 3 -codegen-units = 1 -lto = "fat" -strip = "debuginfo" - -# Profile optimized for minimizing size for wasm builds -# Unfortunately, wasm-pack can't use this directly yet. -# See https://github.com/rustwasm/wasm-pack/issues/1111 -[profile.release-small] -inherits = "release" -opt-level = "z" -codegen-units = 1 diff --git a/automation/set_default_release_profile.py b/automation/set_default_release_profile.py new file mode 100644 index 00000000..e7b5e8a0 --- /dev/null +++ b/automation/set_default_release_profile.py @@ -0,0 +1,43 @@ +import sys +import toml +from pathlib import Path +import copy +def set_default_release_profile(profile_type: str): + # Compute project root path (2 levels up from script location) + script_path = Path(__file__).resolve() + project_root = script_path.parent.parent + cargo_path = project_root / 'Cargo.toml' + + if not cargo_path.exists(): + print(f"Error: Cargo.toml not found at {cargo_path}") + sys.exit(1) + + # Parse toml file + with open(cargo_path) as f: + cargo_config = toml.load(f) + + # Get source profile + source_profile = f'release-{profile_type}' + if source_profile not in cargo_config['profile']: + print(f"Error: profile.{source_profile} not found in Cargo.toml") + sys.exit(1) + + # Replace release profile with selected profile + cargo_config['profile']['release'] = copy.deepcopy(cargo_config['profile'][source_profile]) + del cargo_config['profile']["release"]["inherits"] + + # Write updated config back to file + with open(cargo_path, 'w') as f: + toml.dump(cargo_config, f) + + print(f"Successfully updated release profile with {source_profile} configuration") + +if __name__ == "__main__": + # set_default_release_profile('opt') + + if len(sys.argv) != 2: + print("Usage: python set_default_release_profile.py ") + sys.exit(1) + + set_default_release_profile(sys.argv[1]) + diff --git a/examples/editor-demo/src/index.js b/examples/editor-demo/src/index.js index b7f01072..d440ec9b 100644 --- a/examples/editor-demo/src/index.js +++ b/examples/editor-demo/src/index.js @@ -45,9 +45,27 @@ async function init() { readOnly: true, }); - const hostname = 'http://127.0.0.1:50051'; - let client = new grpcWeb.GrpcWebClientBase({format: "binary"}); - let send_message_grpc = makeGrpcSendMessageFn(client, hostname); + // Add checkbox for toggling VegaFusion server + const container = document.createElement('div'); + container.className = 'pb-3'; // Add padding-bottom + + const serverCheckbox = document.createElement('input'); + serverCheckbox.type = 'checkbox'; + serverCheckbox.id = 'use-server'; + serverCheckbox.checked = false; + serverCheckbox.className = 'form-check-input me-2'; + + const label = document.createElement('label'); + label.htmlFor = 'use-server'; + label.textContent = 'Use VegaFusion Server'; + label.className = 'form-check-label'; + + // Add elements to container + container.appendChild(serverCheckbox); + container.appendChild(label); + + // Insert container before the chart element + document.getElementById('vega-chart').insertAdjacentElement("beforebegin", container); async function update_chart() { try { @@ -60,12 +78,38 @@ async function init() { mode: "vega", }, }; - let chart_handle = await vegaFusionEmbed( - element, - editor.getValue(), - send_message_grpc, - config, - ); + + let chart_handle; + if (serverCheckbox.checked) { + const hostname = 'http://127.0.0.1:50051'; + try { + let client = new grpcWeb.GrpcWebClientBase({format: "binary"}); + let send_message_grpc = makeGrpcSendMessageFn(client, hostname); + + chart_handle = await vegaFusionEmbed( + element, + editor.getValue(), + config, + send_message_grpc, + ); + + } catch (e) { + // Clear the chart and display an error message + element.innerHTML = ` + `; + return; + } + } else { + chart_handle = await vegaFusionEmbed( + element, + editor.getValue(), + config, + ); + } + server_spec_monaco.setValue(JSON.stringify(chart_handle.serverSpec(), null, 2)); client_spec_monaco.setValue(JSON.stringify(chart_handle.clientSpec(), null, 2)); comm_plan_monaco.setValue(JSON.stringify(chart_handle.commPlan(), null, 2)); @@ -75,10 +119,17 @@ async function init() { client_spec_monaco.setValue(""); comm_plan_monaco.setValue(""); console.log(e); - return + return; } } + // Update chart when checkbox changes + serverCheckbox.addEventListener('change', async () => { + let cardElement = document.getElementById('vega-chart').closest('.card'); // Find the parent card + cardElement.classList.toggle('border-success'); + await update_chart(); + }); + // Update chart (with debounce) when editor value changes await update_chart() let content_change_listener = _.debounce(async (content) => { @@ -109,6 +160,8 @@ let flights_spec = { { "name": "source_0", "url": "https://raw.githubusercontent.com/vega/vega-datasets/main/data/flights-2k.json", + // "url": "https://vegafusion-datasets.s3.amazonaws.com/vega/flights_200k.json", + // "url": "https://vegafusion-datasets.s3.amazonaws.com/vega/flights_200k.parquet", "format": {"type": "json", "parse": {"date": "date"}}, "transform": [ { diff --git a/pixi.toml b/pixi.toml index 27ed22f1..0a795fa5 100644 --- a/pixi.toml +++ b/pixi.toml @@ -28,7 +28,7 @@ dev-py = { cmd = [ ] } # Build Python packages -build-py = { cmd = "rm -rf target/wheels && maturin build -m vegafusion-python/Cargo.toml --release --strip --sdist $0" } +build-py = { cmd = "rm -rf target/wheels && maturin build -m vegafusion-python/Cargo.toml --profile release-opt --strip --sdist $0" } fmt-py = { cmd = "ruff format", cwd="vegafusion-python" } lint-fix-py = { cmd = "ruff format && ruff check --fix", cwd="vegafusion-python" } @@ -66,10 +66,19 @@ run-rs-server = "cargo run --release -p vegafusion-server " # Build Wasm install-wasm-toolchain = "python automation/download_rust_target.py wasm32-unknown-unknown" install-wasm-pack = "export CARGO_HOME=$PIXI_PROJECT_ROOT/.pixi/envs/default && cargo install wasm-pack" -build-wasm = { cmd = "cd vegafusion-wasm && rm -rf pkg/ && wasm-pack build --release && node scripts/update-pkg.js && cd pkg/ && npm install", depends_on = [ +build-wasm = { cmd = """ +python automation/set_default_release_profile.py small && +cd vegafusion-wasm && +rm -rf pkg/ && +wasm-pack build --release && +node scripts/update-pkg.js && +cd pkg/ && npm install && +python ../../automation/set_default_release_profile.py dev +""", depends_on = [ "install-wasm-toolchain", "install-wasm-pack", ] } + pack-wasm = { cmd = "cd vegafusion-wasm && wasm-pack pack", depends_on = [ "build-wasm", ] } @@ -80,7 +89,7 @@ build-js-embed = { cmd = "cd javascript/vegafusion-embed && npm install && npm r ] } # VegaFusion Server -build-rs-vegafusion-server = { cmd = "cargo build -p vegafusion-server --release $0" } +build-rs-server = { cmd = "cargo build -p vegafusion-server --profile release-opt $0" } # minio start-minio = "python automation/start_minio.py" diff --git a/vegafusion-common/Cargo.toml b/vegafusion-common/Cargo.toml index a512c851..84d2ee98 100644 --- a/vegafusion-common/Cargo.toml +++ b/vegafusion-common/Cargo.toml @@ -36,6 +36,7 @@ features = ["ipc"] [dependencies.datafusion-common] workspace = true +default-features = false [dependencies.datafusion-expr] workspace = true diff --git a/vegafusion-core/Cargo.toml b/vegafusion-core/Cargo.toml index 8da08080..6ec405aa 100644 --- a/vegafusion-core/Cargo.toml +++ b/vegafusion-core/Cargo.toml @@ -55,6 +55,7 @@ version = "1.6.9" [dependencies.datafusion-common] workspace = true +default-features = false [dependencies.pyo3] workspace = true diff --git a/vegafusion-runtime/Cargo.toml b/vegafusion-runtime/Cargo.toml index 42fb9ef0..a6f9e7a4 100644 --- a/vegafusion-runtime/Cargo.toml +++ b/vegafusion-runtime/Cargo.toml @@ -10,8 +10,16 @@ version = "1.6.9" description = "VegaFusion Runtime" [features] +default = ["fs", "multi-thread", "rustls-tls", "s3", "http", "parquet"] py = ["vegafusion-core/py"] protobuf-src = ["vegafusion-core/protobuf-src"] +s3 = ["object_store/aws"] +http = ["object_store/http"] +http-wasm = [ "object-store-wasm/http"] +fs = ["tokio/fs"] +parquet = ["datafusion-common/parquet", "datafusion/parquet"] +multi-thread = ["tokio/rt-multi-thread"] +rustls-tls = ["reqwest/rustls-tls"] [dependencies] regex = "^1.5.5" @@ -41,13 +49,10 @@ lodepng = "3.6.1" [dependencies.url] workspace = true -[dependencies.serde_json] -workspace = true - -[dependencies.reqwest-middleware] +[dependencies.cfg-if] workspace = true -[dependencies.reqwest-retry] +[dependencies.serde_json] workspace = true [dependencies.async-trait] @@ -61,7 +66,9 @@ workspace = true [dependencies.object_store] workspace = true -features = ["aws", "http"] + +[dependencies.object-store-wasm] +workspace = true [dependencies.chrono] workspace = true @@ -95,9 +102,11 @@ features = ["derive"] [dependencies.datafusion] workspace = true +default-features = false [dependencies.datafusion-common] workspace = true +default-features = false [dependencies.datafusion-expr] workspace = true @@ -122,12 +131,12 @@ workspace = true [dependencies.tokio] workspace = true -features = ["macros", "rt-multi-thread", "fs"] +features = ["macros"] [dependencies.reqwest] workspace = true default-features = false -features = ["rustls-tls"] +features = ["json"] [dev-dependencies.reqwest] workspace = true diff --git a/vegafusion-runtime/src/data/tasks.rs b/vegafusion-runtime/src/data/tasks.rs index 2a6bcf84..2877a0c7 100644 --- a/vegafusion-runtime/src/data/tasks.rs +++ b/vegafusion-runtime/src/data/tasks.rs @@ -11,21 +11,19 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; use vegafusion_core::data::dataset::VegaFusionDataset; +use crate::task_graph::timezone::RuntimeTzConfig; +use crate::transform::pipeline::TransformPipelineUtils; +use cfg_if::cfg_if; use datafusion::datasource::listing::ListingTableUrl; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::execution::options::{ArrowReadOptions, ReadOptions}; -use datafusion::parquet::data_type::AsBytes; -use datafusion::prelude::{CsvReadOptions, DataFrame, ParquetReadOptions, SessionContext}; +use datafusion::prelude::{CsvReadOptions, DataFrame, SessionContext}; use datafusion_common::config::TableOptions; use datafusion_functions::expr_fn::make_date; use std::sync::Arc; -use tokio::io::AsyncReadExt; - -use crate::task_graph::timezone::RuntimeTzConfig; -use crate::transform::pipeline::TransformPipelineUtils; use vegafusion_common::data::scalar::{ScalarValue, ScalarValueHelpers}; -use vegafusion_common::error::{Result, ResultWithContext, ToExternalError, VegaFusionError}; +use vegafusion_common::error::{Result, ResultWithContext, VegaFusionError}; use vegafusion_core::proto::gen::tasks::data_url_task::Url; use vegafusion_core::proto::gen::tasks::scan_url_format; @@ -37,11 +35,8 @@ use vegafusion_core::task_graph::task_value::TaskValue; use crate::data::util::{DataFrameUtils, SessionContextUtils}; use crate::transform::utils::str_to_timestamp; -use object_store::aws::AmazonS3Builder; -use object_store::http::HttpBuilder; -use object_store::{ClientOptions, ObjectStore}; -use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; -use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; + +use object_store::ObjectStore; use vegafusion_common::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use vegafusion_common::column::flat_col; use vegafusion_common::data::table::VegaFusionTable; @@ -50,6 +45,21 @@ use vegafusion_common::datatypes::{is_integer_datatype, is_string_datatype}; use vegafusion_core::proto::gen::transforms::transform::TransformKind; use vegafusion_core::spec::visitors::extract_inline_dataset; +#[cfg(feature = "s3")] +use object_store::aws::AmazonS3Builder; + +#[cfg(feature = "http")] +use object_store::{http::HttpBuilder, ClientOptions}; + +#[cfg(feature = "fs")] +use tokio::io::AsyncReadExt; + +#[cfg(feature = "parquet")] +use {datafusion::prelude::ParquetReadOptions, vegafusion_common::error::ToExternalError}; + +#[cfg(target_arch = "wasm32")] +use object_store_wasm::HttpStore; + pub fn build_compilation_config( input_vars: &[InputVariable], values: &[TaskValue], @@ -150,7 +160,15 @@ impl TaskCall for DataUrlTask { } else if file_type == Some("parquet") || (file_type.is_none() && (url.ends_with(".parquet"))) { - read_parquet(&url, ctx).await? + cfg_if! { + if #[cfg(any(feature = "parquet"))] { + read_parquet(&url, ctx).await? + } else { + return Err(VegaFusionError::internal(format!( + "Enable parquet support by enabling the `parquet` feature flag" + ))) + } + } } else { return Err(VegaFusionError::internal(format!( "Invalid url file extension {url}" @@ -676,43 +694,58 @@ async fn read_json(url: &str, ctx: Arc) -> Result { let child_url = url.strip_prefix(&base_url.to_string()).unwrap(); match store.get(&child_url.into()).await { Ok(get_res) => { - let bytes = get_res.bytes().await?; - let text: Cow = String::from_utf8_lossy(bytes.as_bytes()); + let bytes = get_res.bytes().await?.to_vec(); + let text: Cow = String::from_utf8_lossy(&bytes); serde_json::from_str(text.as_ref())? } Err(e) => { - if url.starts_with("http://") || url.starts_with("https://") { - // Fallback to direct reqwest implementation. This is needed in some cases because - // the object-store http implementation has stricter requirements on what the - // server provides. For example the content-length header is required. - let response = make_request_client() - .get(url) - .send() - .await - .external(format!("Failed to fetch URL: {url}"))?; - - let text = response - .text() - .await - .external("Failed to read response as text")?; - serde_json::from_str(&text)? - } else { - return Err(VegaFusionError::from(e)); + cfg_if::cfg_if! { + if #[cfg(feature="http")] { + if url.starts_with("http://") || url.starts_with("https://") { + // Fallback to direct reqwest implementation. This is needed in some cases because + // the object-store http implementation has stricter requirements on what the + // server provides. For example the content-length header is required. + let client = reqwest::Client::new(); + let response = client + .get(url) + .send() + .await + .external(format!("Failed to fetch URL: {url}"))?; + + let text = response + .text() + .await + .external("Failed to read response as text")?; + serde_json::from_str(&text)? + } else { + return Err(VegaFusionError::from(e)); + } + } else { + return Err(VegaFusionError::from(e)); + } } } } } else { - // Assume local file - let mut file = tokio::fs::File::open(url) - .await - .external(format!("Failed to open as local file: {url}"))?; - - let mut json_str = String::new(); - file.read_to_string(&mut json_str) - .await - .external("Failed to read file contents to string")?; - - serde_json::from_str(&json_str)? + cfg_if::cfg_if! { + if #[cfg(feature="fs")] { + // Assume local file + let mut file = tokio::fs::File::open(url) + .await + .external(format!("Failed to open as local file: {url}"))?; + + let mut json_str = String::new(); + file.read_to_string(&mut json_str) + .await + .external("Failed to read file contents to string")?; + + serde_json::from_str(&json_str)? + } else { + return Err(VegaFusionError::internal( + "The `fs` feature flag must be enabled for file system support" + )); + } + } }; let table = VegaFusionTable::from_json(&value)?.with_ordering()?; @@ -724,6 +757,7 @@ async fn read_arrow(url: &str, ctx: Arc) -> Result { Ok(ctx.read_arrow(url, ArrowReadOptions::default()).await?) } +#[cfg(feature = "parquet")] async fn read_parquet(url: &str, ctx: Arc) -> Result { maybe_register_object_stores_for_url(&ctx, url)?; Ok(ctx.read_parquet(url, ParquetReadOptions::default()).await?) @@ -734,47 +768,57 @@ fn maybe_register_object_stores_for_url( url: &str, ) -> Result> { // Handle object store registration for non-local sources - let maybe_register_http_store = |prefix: &str| -> Result> { - if let Some(path) = url.strip_prefix(prefix) { - let Some((root, _)) = path.split_once('/') else { - return Err(VegaFusionError::specification(format!( - "Invalid https URL: {url}" - ))); - }; - let base_url_str = format!("https://{root}"); - let base_url = url::Url::parse(&base_url_str)?; - - // Register store for url if not already registered - let object_store_url = ObjectStoreUrl::parse(&base_url_str)?; - if ctx - .runtime_env() - .object_store(object_store_url.clone()) - .is_err() - { - let client_options = ClientOptions::new().with_allow_http(true); - let http_store = HttpBuilder::new() - .with_url(base_url.clone()) - .with_client_options(client_options) - .build()?; - - ctx.register_object_store(&base_url, Arc::new(http_store)); + #[cfg(any(feature = "http", feature = "http-wasm"))] + { + let maybe_register_http_store = |prefix: &str| -> Result> { + if let Some(path) = url.strip_prefix(prefix) { + let Some((root, _)) = path.split_once('/') else { + return Err(VegaFusionError::specification(format!( + "Invalid https URL: {url}" + ))); + }; + let base_url_str = format!("https://{root}"); + let base_url = url::Url::parse(&base_url_str)?; + + // Register store for url if not already registered + let object_store_url = ObjectStoreUrl::parse(&base_url_str)?; + if ctx + .runtime_env() + .object_store(object_store_url.clone()) + .is_err() + { + cfg_if! { + if #[cfg(feature="http")] { + let client_options = ClientOptions::new().with_allow_http(true); + let http_store = HttpBuilder::new() + .with_url(base_url.clone()) + .with_client_options(client_options) + .build()?; + ctx.register_object_store(&base_url, Arc::new(http_store)); + } else { + let http_store = HttpStore::new(base_url.clone()); + ctx.register_object_store(&base_url, Arc::new(http_store)); + } + } + } + return Ok(Some(object_store_url)); } - return Ok(Some(object_store_url)); - } - Ok(None) - }; + Ok(None) + }; - // Register https:// - if let Some(url) = maybe_register_http_store("https://")? { - return Ok(Some(url)); - } + // Register https:// + if let Some(url) = maybe_register_http_store("https://")? { + return Ok(Some(url)); + } - // Register http:// - if let Some(url) = maybe_register_http_store("http://")? { - return Ok(Some(url)); + // Register http:// + if let Some(url) = maybe_register_http_store("http://")? { + return Ok(Some(url)); + } } // Register s3:// + #[cfg(feature = "s3")] if let Some(bucket_path) = url.strip_prefix("s3://") { let Some((bucket, _)) = bucket_path.split_once('/') else { return Err(VegaFusionError::specification(format!( @@ -801,11 +845,3 @@ fn maybe_register_object_stores_for_url( Ok(None) } - -pub fn make_request_client() -> ClientWithMiddleware { - // Retry up to 3 times with increasing intervals between attempts. - let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); - ClientBuilder::new(reqwest::Client::new()) - .with(RetryTransientMiddleware::new_with_policy(retry_policy)) - .build() -} diff --git a/vegafusion-runtime/src/datafusion/context.rs b/vegafusion-runtime/src/datafusion/context.rs index 6e461a69..911152a3 100644 --- a/vegafusion-runtime/src/datafusion/context.rs +++ b/vegafusion-runtime/src/datafusion/context.rs @@ -1,16 +1,32 @@ use crate::datafusion::udafs::percentile::{Q1_UDF, Q3_UDF}; use crate::datafusion::udfs::datetime::make_timestamptz::MAKE_UTC_TIMESTAMP; use crate::datafusion::udfs::datetime::timeunit::TIMEUNIT_START_UDF; -use datafusion::execution::runtime_env::RuntimeEnv; +use cfg_if::cfg_if; use datafusion::execution::SessionStateBuilder; -use datafusion::prelude::{SessionConfig, SessionContext}; -use std::sync::Arc; +use datafusion::execution::{config::SessionConfig, runtime_env::RuntimeEnvBuilder}; +use datafusion::prelude::SessionContext; + +#[cfg(target_arch = "wasm32")] +use datafusion::execution::disk_manager::DiskManagerConfig; pub fn make_datafusion_context() -> SessionContext { let mut config = SessionConfig::new(); + let options = config.options_mut(); options.optimizer.skip_failed_rules = true; - let runtime = Arc::new(RuntimeEnv::default()); + + cfg_if! { + if #[cfg(target_arch = "wasm32")] { + // Disable disk manager for wasm runtime since local files aren't supported + let runtime = RuntimeEnvBuilder::new() + .with_disk_manager(DiskManagerConfig::Disabled) + .build_arc() + .unwrap(); + } else { + let runtime = RuntimeEnvBuilder::new().build_arc().unwrap(); + } + } + let session_state = SessionStateBuilder::new() .with_config(config) .with_runtime_env(runtime) diff --git a/vegafusion-runtime/src/task_graph/cache.rs b/vegafusion-runtime/src/task_graph/cache.rs index f539ad90..fd425bd7 100644 --- a/vegafusion-runtime/src/task_graph/cache.rs +++ b/vegafusion-runtime/src/task_graph/cache.rs @@ -2,19 +2,22 @@ use async_lock::{Mutex, MutexGuard, RwLock}; use futures::FutureExt; use lru::LruCache; +use cfg_if::cfg_if; use std::collections::HashMap; use std::future::Future; use std::panic::{resume_unwind, AssertUnwindSafe}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::time::Instant; -use vegafusion_core::error::{DuplicateResult, Result, ToExternalError, VegaFusionError}; +use vegafusion_core::error::{DuplicateResult, Result, VegaFusionError}; use vegafusion_core::task_graph::task_value::TaskValue; +#[cfg(not(target_arch = "wasm32"))] +use {std::time::Instant, vegafusion_core::error::ToExternalError}; + #[derive(Debug, Clone)] struct CachedValue { value: NodeValue, - _calculation_millis: u128, + _calculation_millis: Option, } impl CachedValue { @@ -216,7 +219,12 @@ impl VegaFusionCache { .store(protected.len() + probationary.len(), Ordering::Relaxed); } - async fn set_value(&self, state_fingerprint: u64, value: NodeValue, calculation_millis: u128) { + async fn set_value( + &self, + state_fingerprint: u64, + value: NodeValue, + calculation_millis: Option, + ) { let cache_value = CachedValue { value, _calculation_millis: calculation_millis, @@ -305,52 +313,92 @@ impl VegaFusionCache { .await .insert(state_fingerprint, initializer.clone()); - // Record start time - let start = Instant::now(); - // Invoke future to initialize - match AssertUnwindSafe(tokio::spawn(init)).catch_unwind().await { - // Resolved. - Ok(Ok(value)) => { - // If result Ok, clone to values - match value { + cfg_if! { + if #[cfg(target_arch = "wasm32")] { + // In WASM we await the future directly since multi-threading with tokio::spawn + // is not available. + match AssertUnwindSafe(init).catch_unwind().await { + // Resolved. Ok(value) => { - *initializer_lock = Some(Ok(value.clone())); - - // Check if we should add value to long-term cache - let duration = start.elapsed(); - let millis = duration.as_millis(); - self.set_value(state_fingerprint, value.clone(), millis) - .await; + // If result Ok, clone to values + match value { + Ok(value) => { + *initializer_lock = Some(Ok(value.clone())); + self.set_value(state_fingerprint, value.clone(), None).await; + + // Stored initializer no longer required. Initializers are Arc + // pointers, so it's fine to drop initializer from here even if + // other tasks are still awaiting on it. + self.remove_initializer(state_fingerprint).await; + Ok(value) + } + Err(e) => { + // Remove initializer so that another future can try again + *initializer_lock = Some(Err(e.duplicate())); + self.remove_initializer(state_fingerprint).await; + Err(e) + } + } + } + // Panicked. + Err(payload) => { + *initializer_lock = Some(Err(VegaFusionError::internal("Panic error"))); - // Stored initializer no longer required. Initializers are Arc - // pointers, so it's fine to drop initializer from here even if - // other tasks are still awaiting on it. + // Remove the waiter so that others can retry. self.remove_initializer(state_fingerprint).await; - Ok(value) + // triggers panic, so no return value in this branch + resume_unwind(payload); } - Err(e) => { - // Remove initializer so that another future can try again - *initializer_lock = Some(Err(e.duplicate())); + } + } else { + // When not in WASM, use tokio::spawn for multi-threading + let start = Instant::now(); + match AssertUnwindSafe(tokio::spawn(init)).catch_unwind().await { + // Resolved. + Ok(Ok(value)) => { + // If result Ok, clone to values + match value { + Ok(value) => { + *initializer_lock = Some(Ok(value.clone())); + + // Check if we should add value to long-term cache + let duration = start.elapsed(); + let millis = duration.as_millis(); + + self.set_value(state_fingerprint, value.clone(), Some(millis)) + .await; + + // Stored initializer no longer required. Initializers are Arc + // pointers, so it's fine to drop initializer from here even if + // other tasks are still awaiting on it. + self.remove_initializer(state_fingerprint).await; + Ok(value) + } + Err(e) => { + // Remove initializer so that another future can try again + *initializer_lock = Some(Err(e.duplicate())); + self.remove_initializer(state_fingerprint).await; + Err(e) + } + } + } + Ok(Err(err)) => { + *initializer_lock = Some(Err(VegaFusionError::internal(err.to_string()))); self.remove_initializer(state_fingerprint).await; - Err(e) + Err(err).external("tokio error") + } + // Panicked. + Err(payload) => { + *initializer_lock = Some(Err(VegaFusionError::internal("Panic error"))); + + // Remove the waiter so that others can retry. + self.remove_initializer(state_fingerprint).await; + // triggers panic, so no return value in this branch + resume_unwind(payload); } } } - Ok(Err(err)) => { - *initializer_lock = Some(Err(VegaFusionError::internal(err.to_string()))); - self.remove_initializer(state_fingerprint).await; - Err(err).external("tokio error") - } - // Panicked. - Err(payload) => { - *initializer_lock = Some(Err(VegaFusionError::internal("Panic error"))); - - // Remove the waiter so that others can retry. - self.remove_initializer(state_fingerprint).await; - // triggers panic, so no return value in this branch - resume_unwind(payload); - } } } } diff --git a/vegafusion-runtime/src/task_graph/runtime.rs b/vegafusion-runtime/src/task_graph/runtime.rs index 99003e19..c0932584 100644 --- a/vegafusion-runtime/src/task_graph/runtime.rs +++ b/vegafusion-runtime/src/task_graph/runtime.rs @@ -2,6 +2,7 @@ use crate::task_graph::cache::VegaFusionCache; use crate::task_graph::task::TaskCall; use crate::task_graph::timezone::RuntimeTzConfig; use async_recursion::async_recursion; +use cfg_if::cfg_if; use datafusion::prelude::SessionContext; use futures_util::{future, FutureExt}; use std::any::Any; @@ -159,13 +160,23 @@ async fn get_or_compute_node_value( // Create future to compute node value (will only be executed if not present in cache) let mut inputs_futures = Vec::new(); for input_node_index in input_node_indexes { - inputs_futures.push(tokio::spawn(get_or_compute_node_value( + let node_fut = get_or_compute_node_value( task_graph.clone(), input_node_index, cloned_cache.clone(), inline_datasets.clone(), ctx.clone(), - ))); + ); + + cfg_if! { + if #[cfg(target_arch = "wasm32")] { + // Add future directly + inputs_futures.push(node_fut); + } else { + // In non-wasm environment, use tokio::spawn for multi-threading + inputs_futures.push(tokio::spawn(node_fut)); + } + } } let input_values = futures::future::join_all(inputs_futures).await; @@ -175,13 +186,24 @@ async fn get_or_compute_node_value( .into_iter() .zip(input_edges) .map(|(value, edge)| { - // Convert outer JoinHandle error to internal VegaFusionError so we can propagate it. - let mut value = match value { - Ok(value) => value?, - Err(join_err) => { - return Err(VegaFusionError::internal(join_err.to_string())) + cfg_if! { + if #[cfg(target_arch = "wasm32")] { + let mut value = match value { + Ok(value) => value, + Err(join_err) => { + return Err(join_err) + } + }; + } else { + // Convert outer JoinHandle error to internal VegaFusionError so we can propagate it. + let mut value = match value { + Ok(value) => value?, + Err(join_err) => { + return Err(VegaFusionError::internal(join_err.to_string())) + } + }; } - }; + } let value = match edge.output { None => value.0, diff --git a/vegafusion-runtime/src/tokio_runtime.rs b/vegafusion-runtime/src/tokio_runtime.rs index acdfa892..ed828697 100644 --- a/vegafusion-runtime/src/tokio_runtime.rs +++ b/vegafusion-runtime/src/tokio_runtime.rs @@ -1,12 +1,25 @@ +use cfg_if::cfg_if; use tokio::runtime::Runtime; /// Double default stack size from 2MB to 4MB pub const TOKIO_THREAD_STACK_SIZE: usize = 4 * 1024 * 1024; -lazy_static! { - pub static ref TOKIO_RUNTIME: Runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_stack_size(TOKIO_THREAD_STACK_SIZE) - .build() - .unwrap(); +cfg_if! { + if #[cfg(feature="multi-thread")] { + lazy_static! { + pub static ref TOKIO_RUNTIME: Runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_stack_size(TOKIO_THREAD_STACK_SIZE) + .build() + .unwrap(); + } + } else { + lazy_static! { + pub static ref TOKIO_RUNTIME: Runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .thread_stack_size(TOKIO_THREAD_STACK_SIZE) + .build() + .unwrap(); + } + } } diff --git a/vegafusion-runtime/tests/util/datasets.rs b/vegafusion-runtime/tests/util/datasets.rs index c2e46963..ed3c070f 100644 --- a/vegafusion-runtime/tests/util/datasets.rs +++ b/vegafusion-runtime/tests/util/datasets.rs @@ -1,10 +1,8 @@ -use reqwest_middleware::ClientWithMiddleware; use serde_json::Value; use vegafusion_common::data::table::VegaFusionTable; -use vegafusion_runtime::data::tasks::make_request_client; lazy_static! { - pub static ref REQWEST_CLIENT: ClientWithMiddleware = make_request_client(); + pub static ref REQWEST_CLIENT: reqwest::Client = reqwest::Client::new(); } pub fn vega_json_dataset(name: &str) -> VegaFusionTable { diff --git a/vegafusion-wasm/Cargo.toml b/vegafusion-wasm/Cargo.toml index 257bde10..ca339595 100644 --- a/vegafusion-wasm/Cargo.toml +++ b/vegafusion-wasm/Cargo.toml @@ -39,6 +39,12 @@ version = "1.6.9" path = "../vegafusion-core" version = "1.6.9" +[dependencies.vegafusion-runtime] +path = "../vegafusion-runtime" +version = "1.6.9" +default-features = false +features = ["http-wasm"] + [dependencies.serde] version = "1.0.137" features = ["derive"] diff --git a/vegafusion-wasm/src/lib.rs b/vegafusion-wasm/src/lib.rs index f838b911..e4ae8621 100644 --- a/vegafusion-wasm/src/lib.rs +++ b/vegafusion-wasm/src/lib.rs @@ -31,6 +31,8 @@ use vegafusion_core::spec::chart::ChartSpec; use vegafusion_core::chart_state::ChartState; use vegafusion_core::data::dataset::VegaFusionDataset; +use vegafusion_runtime::datafusion::context::make_datafusion_context; +use vegafusion_runtime::task_graph::runtime::VegaFusionRuntime; use web_sys::Element; fn set_panic_hook() { @@ -52,14 +54,16 @@ extern "C" { fn log(s: &str); } -pub struct VegaFusionWasmRuntime { +/// VegaFusionRuntimeTrait implementation that sends query requests to a VegaFusionRuntime using +/// an async JavaScript function. +pub struct QueryFnVegaFusionRuntime { sender: async_mpsc::Sender<( QueryRequest, oneshot::Sender>>, )>, } -impl VegaFusionWasmRuntime { +impl QueryFnVegaFusionRuntime { pub fn new(query_fn: js_sys::Function) -> Self { let (sender, mut receiver) = async_mpsc::channel::<( QueryRequest, @@ -75,12 +79,58 @@ impl VegaFusionWasmRuntime { let context = JsValue::null(); let js_buffer = js_sys::Uint8Array::from(buf.as_slice()); - let promise = query_fn - .call1(&context, &js_buffer) - .expect("query_fn function call failed"); - let promise = promise.dyn_into::().unwrap(); - let response = JsFuture::from(promise).await.unwrap(); - let response_array = response.dyn_into::().unwrap(); + let promise = match query_fn.call1(&context, &js_buffer) { + Ok(p) => p, + Err(e) => { + response_tx + .send(Err(vegafusion_common::error::VegaFusionError::internal( + format!( + "Failed to call send query functions: {}", + js_sys::JSON::stringify(&e).unwrap() + ), + ))) + .unwrap(); + continue; + } + }; + let promise = match promise.dyn_into::() { + Ok(p) => p, + Err(e) => { + response_tx + .send(Err(vegafusion_common::error::VegaFusionError::internal( + format!( + "send query function did not return a promise: {}", + js_sys::JSON::stringify(&e).unwrap() + ), + ))) + .unwrap(); + continue; + } + }; + let response = match JsFuture::from(promise).await { + Ok(response) => response, + Err(e) => { + response_tx.send(Err(vegafusion_common::error::VegaFusionError::internal( + format!("Error when resolving promise returned by send query function: {}", js_sys::JSON::stringify(&e).unwrap()) + ))).unwrap(); + continue; + } + }; + let response_array = match response.dyn_into::() { + Ok(response_array) => response_array, + Err(e) => { + response_tx + .send(Err(vegafusion_common::error::VegaFusionError::internal( + format!( + "send query function did not return a Uint8Array: {}", + js_sys::JSON::stringify(&e).unwrap() + ), + ))) + .unwrap(); + continue; + } + }; + let response_bytes = response_array.to_vec(); let response = QueryResult::decode(response_bytes.as_slice()).unwrap(); @@ -102,12 +152,12 @@ impl VegaFusionWasmRuntime { } }); - VegaFusionWasmRuntime { sender } + QueryFnVegaFusionRuntime { sender } } } #[async_trait::async_trait] -impl VegaFusionRuntimeTrait for VegaFusionWasmRuntime { +impl VegaFusionRuntimeTrait for QueryFnVegaFusionRuntime { fn as_any(&self) -> &dyn Any { self } @@ -352,29 +402,31 @@ impl Default for VegaFusionEmbedConfig { /// Embed a Vega chart and accelerate with VegaFusion /// @param element - The DOM element to embed the visualization into /// @param spec - The Vega specification (as string or object) -/// @param query_fn - Function to handle server-side query requests /// @param config - Optional configuration options +/// @param query_fn - Function to handle server-side query requests. +/// If not provided, an embedded wasm VegaFusion runtime is created. /// @returns A ChartHandle instance for the embedded visualization #[wasm_bindgen(js_name = vegaFusionEmbed)] pub async fn vegafusion_embed( element: Element, spec: JsValue, - query_fn: js_sys::Function, config: JsValue, -) -> ChartHandle { + query_fn: JsValue, +) -> Result { set_panic_hook(); let spec: ChartSpec = if spec.is_string() { serde_json::from_str(&spec.as_string().unwrap()) - .expect("Failed to convert string to ChartSpec") + .map_err(|_e| JsError::new("Failed to convert JsValue to ChartSpec"))? } else { - serde_wasm_bindgen::from_value(spec).expect("Failed to convert JsValue to ChartSpec") + serde_wasm_bindgen::from_value(spec) + .map_err(|_e| JsError::new("Failed to convert JsValue to ChartSpec"))? }; let config: VegaFusionEmbedConfig = if config.is_undefined() || config.is_null() { VegaFusionEmbedConfig::default() } else { serde_wasm_bindgen::from_value(config) - .expect("Failed to convert JsValue to VegaFusionEmbedConfig") + .map_err(|_e| JsError::new("Failed to convert JsValue to VegaFusionEmbedConfig"))? }; let local_tz = local_timezone(); @@ -383,10 +435,25 @@ pub async fn vegafusion_embed( default_input_tz: None, }; - let runtime = VegaFusionWasmRuntime::new(query_fn); - let chart_state = ChartState::try_new(&runtime, spec, Default::default(), tz_config, None) - .await - .unwrap(); + let runtime: Box = if query_fn.is_undefined() || query_fn.is_null() + { + // Use embedded runtime + let ctx = make_datafusion_context(); + Box::new(VegaFusionRuntime::new(Arc::new(ctx), None, None)) + } else { + let query_fn = query_fn.dyn_into::().map_err(|e| { + JsError::new(&format!( + "Expected query_fn to be a Function: {}", + js_sys::JSON::stringify(&e).unwrap() + )) + })?; + Box::new(QueryFnVegaFusionRuntime::new(query_fn)) + }; + + let chart_state = + ChartState::try_new(runtime.as_ref(), spec, Default::default(), tz_config, None) + .await + .map_err(|e| JsError::new(&e.to_string()))?; // Serializer that can be used to convert serde types to JSON compatible objects let serializer = serde_wasm_bindgen::Serializer::json_compatible(); @@ -403,7 +470,12 @@ pub async fn vegafusion_embed( .serialize(&serializer) .expect("Failed to convert embed_opts to JsValue"); - let embed = embed(element, spec_value, opts).await.unwrap(); + let embed = embed(element, spec_value, opts).await.map_err(|e| { + JsError::new(&format!( + "Failed to embed chart: {}", + js_sys::JSON::stringify(&e).unwrap() + )) + })?; let (sender, mut receiver) = async_mpsc::channel::(16); @@ -422,16 +494,17 @@ pub async fn vegafusion_embed( // listen for callback updates spawn_local(async move { while let Some(update) = receiver.next().await { - let response_update = inner_handle + if let Ok(response_update) = inner_handle .state - .update(&runtime, vec![update]) + .update(runtime.as_ref(), vec![update]) .await - .unwrap(); - inner_handle.update_view(&response_update); + { + inner_handle.update_view(&response_update); + } } }); - handle + Ok(handle) } /// Create a function for sending VegaFusion queries to VegaFusion server over gRPC-Web