diff --git a/.cargo/config.toml b/.cargo/config.toml index 02ec924428..99e7fe0b0b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,6 +2,4 @@ rustflags = [ "-C", "target-feature=-crt-static", - "-C", - "link-arg=/FORCE:MULTIPLE", ] diff --git a/Cargo.lock b/Cargo.lock index 2c59bb438b..f1e9cad529 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,7 +64,7 @@ dependencies = [ "pin-project-lite", "rand 0.9.1", "sha1", - "smallvec", + "smallvec 1.15.0", "tokio", "tokio-util", "tracing", @@ -163,7 +163,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "smallvec", + "smallvec 1.15.0", "socket2", "time", "tracing", @@ -596,7 +596,7 @@ dependencies = [ "objc2-foundation 0.3.1", "parking_lot 0.12.3", "percent-encoding", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "wl-clipboard-rs", "x11rb", ] @@ -1758,7 +1758,7 @@ dependencies = [ "bitflags 2.9.1", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -2358,7 +2358,7 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" dependencies = [ - "smallvec", + "smallvec 1.15.0", "target-lexicon", ] @@ -2699,7 +2699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2708,7 +2708,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3124,7 +3124,7 @@ dependencies = [ "phf 0.10.1", "proc-macro2", "quote", - "smallvec", + "smallvec 1.15.0", "syn 1.0.109", ] @@ -3537,6 +3537,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "der-parser" version = "8.2.0" @@ -3992,7 +4002,7 @@ dependencies = [ "bitflags 1.3.2", "dasp_frame", "dasp_sample", - "smallvec", + "smallvec 1.15.0", ] [[package]] @@ -4001,7 +4011,7 @@ version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" dependencies = [ - "der", + "der 0.6.1", "elliptic-curve", "rfc6979", "signature", @@ -4044,7 +4054,7 @@ checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" dependencies = [ "base16ct", "crypto-bigint 0.4.9", - "der", + "der 0.6.1", "digest 0.10.7", "ff", "generic-array", @@ -4153,7 +4163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5109,7 +5119,7 @@ dependencies = [ "libc", "once_cell", "pin-project-lite", - "smallvec", + "smallvec 1.15.0", "thiserror 1.0.69", ] @@ -5174,7 +5184,7 @@ dependencies = [ "once_cell", "parking_lot 0.12.3", "signal-hook", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", ] @@ -5204,7 +5214,7 @@ dependencies = [ "gix-quote", "gix-trace", "kstring", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", "unicode-bom", ] @@ -5268,7 +5278,7 @@ dependencies = [ "gix-sec", "memchr", "once_cell", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", "unicode-bom", "winnow 0.7.10", @@ -5406,7 +5416,7 @@ dependencies = [ "gix-quote", "gix-trace", "gix-utils 0.2.0", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", ] @@ -5522,7 +5532,7 @@ dependencies = [ "libc", "memmap2", "rustix 0.38.44", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", ] @@ -5553,7 +5563,7 @@ dependencies = [ "gix-utils 0.2.0", "gix-validate 0.9.4", "itoa", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", "winnow 0.7.10", ] @@ -5593,7 +5603,7 @@ dependencies = [ "gix-object", "gix-path", "memmap2", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", ] @@ -5711,7 +5721,7 @@ dependencies = [ "gix-hash 0.17.0", "gix-revision", "gix-validate 0.9.4", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", ] @@ -5744,7 +5754,7 @@ dependencies = [ "gix-hash 0.17.0", "gix-hashtable", "gix-object", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", ] @@ -5861,7 +5871,7 @@ dependencies = [ "gix-hashtable", "gix-object", "gix-revwalk", - "smallvec", + "smallvec 1.15.0", "thiserror 2.0.12", ] @@ -5958,7 +5968,7 @@ dependencies = [ "libc", "memchr", "once_cell", - "smallvec", + "smallvec 1.15.0", "thiserror 1.0.69", ] @@ -6465,7 +6475,7 @@ dependencies = [ "serde", "serde_json", "thiserror 1.0.69", - "ureq", + "ureq 2.12.1", ] [[package]] @@ -6692,7 +6702,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "smallvec", + "smallvec 1.15.0", "tokio", "want", ] @@ -6866,7 +6876,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.59.0", + "windows-core 0.61.2", ] [[package]] @@ -6925,7 +6935,7 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", - "smallvec", + "smallvec 1.15.0", "zerovec", ] @@ -6987,7 +6997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ "idna_adapter", - "smallvec", + "smallvec 1.15.0", "utf8_iter", ] @@ -7277,7 +7287,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7843,7 +7853,7 @@ dependencies = [ "fallible-streaming-iterator", "hashlink", "libsql-ffi", - "smallvec", + "smallvec 1.15.0", ] [[package]] @@ -9665,27 +9675,28 @@ dependencies = [ [[package]] name = "ort" -version = "2.0.0-rc.9" +version = "2.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52afb44b6b0cffa9bf45e4d37e5a4935b0334a51570658e279e9e3e6cf324aa5" +checksum = "1fa7e49bd669d32d7bc2a15ec540a527e7764aec722a45467814005725bcd721" dependencies = [ - "half", + "libloading 0.8.7", "ndarray", "ort-sys", + "smallvec 2.0.0-alpha.10", "tracing", ] [[package]] name = "ort-sys" -version = "2.0.0-rc.9" +version = "2.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41d7757331aef2d04b9cb09b45583a59217628beaf91895b7e76187b6e8c088" +checksum = "e2aba9f5c7c479925205799216e7e5d07cc1d4fa76ea8058c60a9a30f6a4e890" dependencies = [ "flate2", "pkg-config", "sha2 0.10.9", "tar", - "ureq", + "ureq 3.0.12", ] [[package]] @@ -9808,7 +9819,7 @@ dependencies = [ "instant", "libc", "redox_syscall 0.2.16", - "smallvec", + "smallvec 1.15.0", "winapi", ] @@ -9821,7 +9832,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall 0.5.12", - "smallvec", + "smallvec 1.15.0", "windows-targets 0.52.6", ] @@ -9907,6 +9918,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -10172,7 +10192,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" dependencies = [ - "der", + "der 0.6.1", "spki", ] @@ -10457,7 +10477,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", @@ -10478,7 +10498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.101", @@ -10653,7 +10673,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11505,7 +11525,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11518,7 +11538,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11855,7 +11875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" dependencies = [ "base16ct", - "der", + "der 0.6.1", "generic-array", "pkcs8", "subtle", @@ -11923,7 +11943,7 @@ dependencies = [ "phf_codegen 0.8.0", "precomputed-hash", "servo_arc", - "smallvec", + "smallvec 1.15.0", ] [[package]] @@ -11957,7 +11977,7 @@ dependencies = [ "sentry-panic 0.36.0", "sentry-tracing 0.36.0", "tokio", - "ureq", + "ureq 2.12.1", ] [[package]] @@ -11977,7 +11997,7 @@ dependencies = [ "sentry-panic 0.38.1", "sentry-tracing 0.38.1", "tokio", - "ureq", + "ureq 2.12.1", ] [[package]] @@ -12778,6 +12798,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +[[package]] +name = "smallvec" +version = "2.0.0-alpha.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d44cfb396c3caf6fbfd0ab422af02631b69ddd96d2eff0b0f0724f9024051b" + [[package]] name = "smart-default" version = "0.6.0" @@ -12946,7 +12972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" dependencies = [ "base64ct", - "der", + "der 0.6.1", ] [[package]] @@ -14515,7 +14541,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -15297,7 +15323,7 @@ dependencies = [ "serde", "serde_json", "sharded-slab", - "smallvec", + "smallvec 1.15.0", "thread_local", "tracing", "tracing-core", @@ -15663,7 +15689,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" dependencies = [ - "smallvec", + "smallvec 1.15.0", ] [[package]] @@ -15727,11 +15753,41 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "socks", "url", "webpki-roots 0.26.11", ] +[[package]] +name = "ureq" +version = "3.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f0fde9bc91026e381155f8c67cb354bcd35260b2f4a29bcc84639f762760c39" +dependencies = [ + "base64 0.22.1", + "der 0.7.10", + "log", + "native-tls", + "percent-encoding", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf-8", + "webpki-root-certs 0.26.11", +] + +[[package]] +name = "ureq-proto" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59db78ad1923f2b1be62b6da81fe80b173605ca0d57f85da2e005382adf693f7" +dependencies = [ + "base64 0.22.1", + "http 1.3.1", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.4" @@ -15920,15 +15976,16 @@ dependencies = [ [[package]] name = "wasapi" -version = "0.16.0" -source = "git+https://github.com/HEnquist/wasapi-rs?rev=24ae99c0134f7e1429d79ba3105a4f796e92ee6d#24ae99c0134f7e1429d79ba3105a4f796e92ee6d" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44f8b836526917dd6b4ca0ca163394df7cadae249ee6a8b10d36ee4265298c9" dependencies = [ "log", "num-integer", "thiserror 2.0.12", "widestring", - "windows 0.59.0", - "windows-core 0.59.0", + "windows 0.61.1", + "windows-core 0.61.2", ] [[package]] @@ -16060,7 +16117,7 @@ dependencies = [ "cc", "downcast-rs", "rustix 0.38.44", - "smallvec", + "smallvec 1.15.0", "wayland-sys", ] @@ -16185,6 +16242,24 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webpki-root-certs" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" +dependencies = [ + "webpki-root-certs 1.0.1", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86138b15b2b7d561bc4469e77027b8dd005a43dc502e9031d1f5afc8ce1f280e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -16416,7 +16491,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -16483,16 +16558,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1" -dependencies = [ - "windows-core 0.59.0", - "windows-targets 0.53.0", -] - [[package]] name = "windows" version = "0.61.1" @@ -16537,19 +16602,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce" -dependencies = [ - "windows-implement 0.59.0", - "windows-interface 0.59.1", - "windows-result 0.3.4", - "windows-strings 0.3.1", - "windows-targets 0.53.0", -] - [[package]] name = "windows-core" version = "0.61.2" @@ -16585,17 +16637,6 @@ dependencies = [ "syn 2.0.101", ] -[[package]] -name = "windows-implement" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.101", -] - [[package]] name = "windows-implement" version = "0.60.0" diff --git a/apps/desktop/src-tauri/.cargo/debug-config.toml b/apps/desktop/src-tauri/.cargo/debug-config.toml new file mode 100644 index 0000000000..4e1bc42950 --- /dev/null +++ b/apps/desktop/src-tauri/.cargo/debug-config.toml @@ -0,0 +1,27 @@ +[target.'cfg(all(windows, target_env = "msvc"))'] +rustflags = [ + "-C", + "target-feature=-crt-static", + "-C", + "link-arg=/nodefaultlib:msvcrt", + "-C", + "link-arg=/defaultlib:msvcrtd", +] + +[env] +# Force all C/C++ compilation to use debug runtime +CFLAGS = "/MDd /D_DEBUG /Od" +CXXFLAGS = "/MDd /D_DEBUG /Od" + +# CMake configuration for debug runtime +CMAKE_CXX_FLAGS = "/MDd /D_DEBUG /Od" +CMAKE_C_FLAGS = "/MDd /D_DEBUG /Od" +CMAKE_MSVC_RUNTIME_LIBRARY = "MultiThreadedDebugDLL" +CMAKE_BUILD_TYPE = "Debug" + +# Global debug flags +DEBUG = "1" +_DEBUG = "1" + +# ORG +ORT_DYLIB_PATH = "./dlls/onnxruntime.dll" diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index e5167be6bf..0132d75383 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -98,6 +98,8 @@ hound = { workspace = true } macos-default = ["llm-metal", "stt-metal", "stt-coreml"] windows-default = ["llm-vulkan", "llm-native", "stt-vulkan", "stt-openblas", "stt-directml"] +load-dynamic = ["tauri-plugin-local-stt/load-dynamic", "tauri-plugin-listener/load-dynamic"] + llm-metal = ["tauri-plugin-local-llm/metal"] llm-cuda = ["tauri-plugin-local-llm/cuda"] llm-vulkan = ["tauri-plugin-local-llm/vulkan"] diff --git a/apps/desktop/src-tauri/dlls/onnxruntime.dll b/apps/desktop/src-tauri/dlls/onnxruntime.dll new file mode 100644 index 0000000000..8fd1f9cdfd Binary files /dev/null and b/apps/desktop/src-tauri/dlls/onnxruntime.dll differ diff --git a/crates/aec/Cargo.toml b/crates/aec/Cargo.toml index 520bbc8123..a5f60a72e9 100644 --- a/crates/aec/Cargo.toml +++ b/crates/aec/Cargo.toml @@ -8,6 +8,7 @@ default = ["256"] 128 = [] 256 = [] 512 = [] +load-dynamic = ["hypr-onnx/load-dynamic"] [dependencies] hypr-onnx = { workspace = true } diff --git a/crates/aec/src/lib.rs b/crates/aec/src/lib.rs index 3b9f24e149..3d9f1d5efc 100644 --- a/crates/aec/src/lib.rs +++ b/crates/aec/src/lib.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use hypr_onnx::{ ndarray::{Array3, Array4}, - ort::session::Session, + ort::{session::Session, value::TensorRef}, }; mod error; @@ -173,15 +173,15 @@ impl AEC { lpb_mag: &Array3, ) -> Result, crate::Error> { let mut outputs = self.session_1.run(hypr_onnx::ort::inputs![ - in_mag.view(), - self.states_1.view(), - lpb_mag.view() - ]?)?; + TensorRef::from_array_view(in_mag.view())?, + TensorRef::from_array_view(self.states_1.view())?, + TensorRef::from_array_view(lpb_mag.view())? + ])?; let out_mask = outputs .remove("Identity") .ok_or_else(|| Error::MissingOutput("Identity".to_string()))? - .try_extract_tensor::()? + .try_extract_array::()? .view() .to_owned(); let out_mask_1d = out_mask.into_shape_with_order((self.block_len / 2 + 1,))?; @@ -189,7 +189,7 @@ impl AEC { self.states_1 = outputs .remove("Identity_1") .ok_or_else(|| Error::MissingOutput("Identity_1".to_string()))? - .try_extract_tensor::()? + .try_extract_array::()? .view() .to_owned() .into_shape_with_order((1, 2, model::STATE_SIZE, 2))?; @@ -203,15 +203,15 @@ impl AEC { in_lpb: &Array3, ) -> Result, crate::Error> { let mut outputs = self.session_2.run(hypr_onnx::ort::inputs![ - estimated_block.view(), - self.states_2.view(), - in_lpb.view() - ]?)?; + TensorRef::from_array_view(estimated_block.view())?, + TensorRef::from_array_view(self.states_2.view())?, + TensorRef::from_array_view(in_lpb.view())? + ])?; let out_block = outputs .remove("Identity") .ok_or_else(|| Error::MissingOutput("Identity".into()))? - .try_extract_tensor::()? + .try_extract_array::()? .view() .to_owned(); let out_block_1d = out_block.into_shape_with_order((self.block_len,))?; @@ -219,7 +219,7 @@ impl AEC { self.states_2 = outputs .remove("Identity_1") .ok_or_else(|| Error::MissingOutput("Identity_1".into()))? - .try_extract_tensor::()? + .try_extract_array::()? .view() .to_owned() .into_shape_with_order((1, 2, model::STATE_SIZE, 2))?; diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index b4011264ec..02414be363 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -27,7 +27,7 @@ ringbuf = { workspace = true } cidre = { workspace = true, features = ["av"] } [target.'cfg(target_os = "windows")'.dependencies] -wasapi = { git = "https://github.com/HEnquist/wasapi-rs", rev = "24ae99c0134f7e1429d79ba3105a4f796e92ee6d" } +wasapi = "0.19.0" [target.'cfg(target_os = "linux")'.dependencies] alsa = "0.9.1" diff --git a/crates/audio/src/speaker/mod.rs b/crates/audio/src/speaker/mod.rs index cb9f10018c..6b48502d26 100644 --- a/crates/audio/src/speaker/mod.rs +++ b/crates/audio/src/speaker/mod.rs @@ -29,13 +29,13 @@ pub struct SpeakerInput { } impl SpeakerInput { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "windows"))] pub fn new(sample_rate_override: Option) -> Result { let inner = PlatformSpeakerInput::new(sample_rate_override)?; Ok(Self { inner }) } - #[cfg(not(any(target_os = "macos")))] + #[cfg(not(any(target_os = "macos", target_os = "windows")))] pub fn new(sample_rate_override: Option) -> Result { Err(anyhow::anyhow!( "'SpeakerInput::new' is not supported on this platform" @@ -151,9 +151,44 @@ mod tests { } #[cfg(target_os = "windows")] - #[test] + #[tokio::test] #[serial] - fn test_windows() { - assert!(true); + async fn test_windows() { + use kalosm_sound::AsyncSource; + + // Test that we can create a SpeakerInput + let input = match SpeakerInput::new(None) { + Ok(input) => input, + Err(e) => { + println!("Failed to create SpeakerInput: {}", e); + return; // Skip test if WASAPI is not available + } + }; + + // Test that we can create a stream + let mut stream = match input.stream() { + Ok(stream) => stream, + Err(e) => { + println!("Failed to create speaker stream: {}", e); + return; + } + }; + + // Check that we get a reasonable sample rate + let sample_rate = stream.sample_rate(); + assert!(sample_rate > 0); + println!("Windows speaker sample rate: {}", sample_rate); + + // Try to get some samples + let mut sample_count = 0; + while let Some(_sample) = stream.next().await { + sample_count += 1; + if sample_count > 100 { + break; + } + } + + assert!(sample_count > 0, "Should receive some audio samples"); + println!("Received {} samples from Windows speaker", sample_count); } } diff --git a/crates/audio/src/speaker/windows.rs b/crates/audio/src/speaker/windows.rs index 213b30e723..4804e30aba 100644 --- a/crates/audio/src/speaker/windows.rs +++ b/crates/audio/src/speaker/windows.rs @@ -1,36 +1,225 @@ +use anyhow::Result; use futures_util::Stream; +use std::collections::VecDeque; +use std::sync::{mpsc, Arc, Mutex}; +use std::task::{Poll, Waker}; +use std::thread; +use std::time::Duration; +use tracing::error; +use wasapi::{get_default_device, Direction, SampleType, StreamMode, WaveFormat}; -pub struct SpeakerInput {} +pub struct SpeakerInput { + sample_rate_override: Option, +} impl SpeakerInput { - pub fn new(_sample_rate_override: Option) -> Self { - Self {} + pub fn new(sample_rate_override: Option) -> Result { + Ok(Self { + sample_rate_override, + }) } pub fn stream(self) -> SpeakerStream { - SpeakerStream::new() + let sample_queue = Arc::new(Mutex::new(VecDeque::new())); + let waker_state = Arc::new(Mutex::new(WakerState { + waker: None, + has_data: false, + shutdown: false, + })); + + let queue_clone = sample_queue.clone(); + let waker_clone = waker_state.clone(); + let (init_tx, init_rx) = mpsc::channel(); + + let capture_thread = thread::spawn(move || { + if let Err(e) = SpeakerStream::capture_audio_loop(queue_clone, waker_clone, init_tx) { + error!("Audio capture loop failed: {}", e); + } + }); + + if let Ok(Err(e)) = init_rx.recv_timeout(Duration::from_secs(5)) { + error!("Audio initialization failed: {}", e); + } + + SpeakerStream { + sample_queue, + waker_state, + capture_thread: Some(capture_thread), + sample_rate_override: self.sample_rate_override, + } } } -pub struct SpeakerStream {} +struct WakerState { + waker: Option, + has_data: bool, + shutdown: bool, +} + +pub struct SpeakerStream { + sample_queue: Arc>>, + waker_state: Arc>, + capture_thread: Option>, + sample_rate_override: Option, +} impl SpeakerStream { - pub fn new() -> Self { - Self {} + pub fn sample_rate(&self) -> u32 { + self.sample_rate_override.unwrap_or(44100) } - pub fn sample_rate(&self) -> u32 { - 16000 + fn capture_audio_loop( + sample_queue: Arc>>, + waker_state: Arc>, + init_tx: mpsc::Sender>, + ) -> Result<()> { + let init_result = (|| -> Result<_> { + let device = get_default_device(&Direction::Render)?; + let mut audio_client = device.get_iaudioclient()?; + + let desired_format = WaveFormat::new(32, 32, &SampleType::Float, 44100, 1, None); + + let (_def_time, min_time) = audio_client.get_device_period()?; + + let mode = StreamMode::EventsShared { + autoconvert: true, + buffer_duration_hns: min_time, + }; + + audio_client.initialize_client(&desired_format, &Direction::Capture, &mode)?; + + let h_event = audio_client.set_get_eventhandle()?; + let render_client = audio_client.get_audiocaptureclient()?; + + audio_client.start_stream()?; + + Ok((h_event, render_client)) + })(); + + match init_result { + Ok((h_event, render_client)) => { + let _ = init_tx.send(Ok(())); + + loop { + { + let state = waker_state.lock().unwrap(); + if state.shutdown { + break; + } + } + + if h_event.wait_for_event(3000).is_err() { + error!("timeout error, stopping capture"); + break; + } + + let mut temp_queue = VecDeque::new(); + if let Err(e) = render_client.read_from_device_to_deque(&mut temp_queue) { + error!("Failed to read audio data: {}", e); + continue; + } + + if temp_queue.is_empty() { + continue; + } + + let mut samples = Vec::new(); + while temp_queue.len() >= 4 { + let bytes = [ + temp_queue.pop_front().unwrap(), + temp_queue.pop_front().unwrap(), + temp_queue.pop_front().unwrap(), + temp_queue.pop_front().unwrap(), + ]; + let sample = f32::from_le_bytes(bytes); + samples.push(sample); + } + + if !samples.is_empty() { + { + let mut queue = sample_queue.lock().unwrap(); + queue.extend(samples); + + let len = queue.len(); + if len > 8192 { + queue.drain(0..(len - 8192)); + } + } + + { + let mut state = waker_state.lock().unwrap(); + if !state.has_data { + state.has_data = true; + if let Some(waker) = state.waker.take() { + drop(state); + waker.wake(); + } + } + } + } + } + } + Err(e) => { + let _ = init_tx.send(Err(e)); + return Ok(()); + } + } + + Ok(()) + } +} + +impl Drop for SpeakerStream { + fn drop(&mut self) { + { + let mut state = self.waker_state.lock().unwrap(); + state.shutdown = true; + } + + if let Some(thread) = self.capture_thread.take() { + if let Err(e) = thread.join() { + error!("Failed to join capture thread: {:?}", e); + } + } } } impl Stream for SpeakerStream { type Item = f32; - fn poll_next( self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::task::Poll::Ready(Some(0.0)) + cx: &mut std::task::Context<'_>, + ) -> Poll> { + { + let state = self.waker_state.lock().unwrap(); + if state.shutdown { + return Poll::Ready(None); + } + } + + { + let mut queue = self.sample_queue.lock().unwrap(); + if let Some(sample) = queue.pop_front() { + return Poll::Ready(Some(sample)); + } + } + + { + let mut state = self.waker_state.lock().unwrap(); + if state.shutdown { + return Poll::Ready(None); + } + state.has_data = false; + state.waker = Some(cx.waker().clone()); + drop(state); + } + + { + let mut queue = self.sample_queue.lock().unwrap(); + match queue.pop_front() { + Some(sample) => Poll::Ready(Some(sample)), + None => Poll::Pending, + } + } } } diff --git a/crates/calendar-apple/Cargo.toml b/crates/calendar-apple/Cargo.toml index 82a049faf8..8fdfe12d8c 100644 --- a/crates/calendar-apple/Cargo.toml +++ b/crates/calendar-apple/Cargo.toml @@ -4,17 +4,17 @@ version = "0.1.0" edition = "2021" [dependencies] +anyhow = { workspace = true } +chrono = { workspace = true } hypr-calendar-interface = { path = "../calendar-interface", package = "calendar-interface" } +itertools = { workspace = true } +[target.'cfg(target_os = "macos")'.dependencies] block2 = "0.5.1" objc2 = "0.5.2" objc2-contacts = { version = "0.2.2", features = ["CNContactStore", "CNLabeledValue", "CNContact", "block2"] } objc2-event-kit = { version = "0.2.2", features = ["EKEventStore", "EKCalendarItem", "EKCalendar", "EKParticipant", "EKObject", "EKEvent", "EKSource", "EKTypes", "block2"] } objc2-foundation = { version = "0.2.2", features = ["NSEnumerator"] } -anyhow = { workspace = true } -chrono = { workspace = true } -itertools = { workspace = true } - [dev-dependencies] tokio = { workspace = true, features = ["rt", "macros"] } diff --git a/crates/chunker/Cargo.toml b/crates/chunker/Cargo.toml index ea862bc808..fcf38d8f21 100644 --- a/crates/chunker/Cargo.toml +++ b/crates/chunker/Cargo.toml @@ -16,3 +16,7 @@ futures-util = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[features] +default = [] +load-dynamic = ["hypr-vad/load-dynamic"] diff --git a/crates/denoise/Cargo.toml b/crates/denoise/Cargo.toml index 433df6b23c..4d976c0628 100644 --- a/crates/denoise/Cargo.toml +++ b/crates/denoise/Cargo.toml @@ -3,6 +3,10 @@ name = "denoise" version = "0.1.0" edition = "2021" +[features] +default = [] +load-dynamic = ["hypr-onnx/load-dynamic"] + [dependencies] hypr-onnx = { workspace = true } diff --git a/crates/onnx/Cargo.toml b/crates/onnx/Cargo.toml index 93ab45a3ef..b1d6c58d16 100644 --- a/crates/onnx/Cargo.toml +++ b/crates/onnx/Cargo.toml @@ -7,7 +7,8 @@ edition = "2021" default = [] coreml = ["ort/coreml"] directml = ["ort/directml"] +load-dynamic = ["ort/load-dynamic"] [dependencies] ndarray = "0.16" -ort = { version = "=2.0.0-rc.9" } +ort = { version = "=2.0.0-rc.10", features = ["ndarray"] } diff --git a/crates/pyannote-local/Cargo.toml b/crates/pyannote-local/Cargo.toml index 7832adc891..e6ff5178c4 100644 --- a/crates/pyannote-local/Cargo.toml +++ b/crates/pyannote-local/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" default = [] coreml = ["hypr-onnx/coreml"] directml = ["hypr-onnx/directml"] +load-dynamic = ["hypr-onnx/load-dynamic"] [dependencies] hypr-onnx = { workspace = true } diff --git a/crates/pyannote-local/src/embedding.rs b/crates/pyannote-local/src/embedding.rs index b5cdbc3b0f..0e6e6a2608 100644 --- a/crates/pyannote-local/src/embedding.rs +++ b/crates/pyannote-local/src/embedding.rs @@ -2,7 +2,7 @@ use dasp::sample::ToSample; use hypr_onnx::{ ndarray::{self, Array2}, - ort::{self, session::Session}, + ort::{self, session::Session, value::TensorRef}, }; const EMBEDDING_ONNX: &[u8] = include_bytes!("./data/embedding.onnx"); @@ -27,10 +27,10 @@ impl EmbeddingExtractor { .map_err(|s| crate::Error::KnfError(s.to_string()))?; let features = features.insert_axis(ndarray::Axis(0)); - let inputs = ort::inputs! ["feats" => features.view()]?; + let inputs = ort::inputs! ["feats" => TensorRef::from_array_view(features.view())?]; let ort_outs = self.session.run(inputs)?; - let ort_out = ort_outs.get("embs").unwrap().try_extract_tensor::()?; + let ort_out = ort_outs.get("embs").unwrap().try_extract_array::()?; let embeddings = ort_out.iter().copied().collect::>(); Ok(embeddings) diff --git a/crates/pyannote-local/src/segmentation.rs b/crates/pyannote-local/src/segmentation.rs index 004c4d55ea..3da3564d4d 100644 --- a/crates/pyannote-local/src/segmentation.rs +++ b/crates/pyannote-local/src/segmentation.rs @@ -1,6 +1,6 @@ use hypr_onnx::{ ndarray::{self, ArrayBase, Axis, IxDyn, ViewRepr}, - ort::{self, session::Session}, + ort::{self, session::Session, value::TensorRef}, }; const SEGMENTATION_ONNX: &[u8] = include_bytes!("./data/segmentation.onnx"); @@ -47,12 +47,12 @@ impl Segmenter { .insert_axis(Axis(1)) .into_dyn(); - let inputs = ort::inputs![array]?; + let inputs = ort::inputs![TensorRef::from_array_view(array.view())?]; let run_output = self.session.run(inputs)?; let output_tensor = run_output.values().next().unwrap(); - let outputs = output_tensor.try_extract_tensor::()?; + let outputs = output_tensor.try_extract_array::()?; - self.process_outputs( + Self::process_outputs( outputs, &mut is_speaking, &mut start_offset, @@ -64,7 +64,7 @@ impl Segmenter { } if is_speaking { - self.create_segment(start_offset, offset, sample_rate, &padded, &mut segments)?; + Self::create_segment(start_offset, offset, sample_rate, &padded, &mut segments)?; } Ok(segments) @@ -82,7 +82,6 @@ impl Segmenter { } fn process_outputs( - &self, outputs: ArrayBase, IxDyn>, is_speaking: &mut bool, start_offset: &mut f64, @@ -93,7 +92,7 @@ impl Segmenter { ) -> Result<(), crate::Error> { for row in outputs.outer_iter() { for sub_row in row.axis_iter(Axis(0)) { - let max_index = self.find_max_index(sub_row)?; + let max_index = Self::find_max_index(sub_row)?; if max_index != 0 { if !*is_speaking { @@ -101,7 +100,7 @@ impl Segmenter { *is_speaking = true; } } else if *is_speaking { - self.create_segment( + Self::create_segment( *start_offset, *offset, sample_rate, @@ -117,7 +116,7 @@ impl Segmenter { Ok(()) } - fn find_max_index(&self, row: ArrayBase, IxDyn>) -> Result { + fn find_max_index(row: ArrayBase, IxDyn>) -> Result { row.iter() .enumerate() .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) @@ -126,7 +125,6 @@ impl Segmenter { } fn create_segment( - &self, start_offset: f64, end_offset: usize, sample_rate: u32, diff --git a/crates/tcc/Cargo.toml b/crates/tcc/Cargo.toml index 60c3b3de17..1e3195ad38 100644 --- a/crates/tcc/Cargo.toml +++ b/crates/tcc/Cargo.toml @@ -3,8 +3,8 @@ name = "tcc" version = "0.1.0" edition = "2021" -[build-dependencies] +[target.'cfg(target_os = "macos")'.build-dependencies] swift-rs = { git = "https://github.com/guillemcordoba/swift-rs", rev = "01980f981bc642a6da382cc0788f18fdd4cde6df", features = ["build"] } -[dependencies] +[target.'cfg(target_os = "macos")'.dependencies] swift-rs = { git = "https://github.com/guillemcordoba/swift-rs", rev = "01980f981bc642a6da382cc0788f18fdd4cde6df" } diff --git a/crates/tcc/build.rs b/crates/tcc/build.rs index a33d3db872..1d4cdfed39 100644 --- a/crates/tcc/build.rs +++ b/crates/tcc/build.rs @@ -1,5 +1,13 @@ fn main() { - swift_rs::SwiftLinker::new("14.2") - .with_package("swift-lib", "./swift-lib/") - .link(); + #[cfg(target_os = "macos")] + { + swift_rs::SwiftLinker::new("14.2") + .with_package("swift-lib", "./swift-lib/") + .link(); + } + + #[cfg(not(target_os = "macos"))] + { + println!("cargo:warning=Swift linking is only available on macOS"); + } } diff --git a/crates/tcc/src/lib.rs b/crates/tcc/src/lib.rs index 200e9cf504..d544f01c52 100644 --- a/crates/tcc/src/lib.rs +++ b/crates/tcc/src/lib.rs @@ -1,14 +1,26 @@ +#[cfg(target_os = "macos")] use swift_rs::{swift, Bool}; +#[cfg(target_os = "macos")] swift!(fn _audio_capture_permission_granted() -> Bool); +#[cfg(not(target_os = "macos"))] +pub fn _audio_capture_permission_granted() -> bool { + true +} + #[cfg(test)] mod tests { use super::*; #[test] fn test_audio_capture_permission_granted() { + #[cfg(target_os = "macos")] let result = unsafe { _audio_capture_permission_granted() }; + + #[cfg(not(target_os = "macos"))] + let result = _audio_capture_permission_granted(); + assert!(result); } } diff --git a/crates/vad/Cargo.toml b/crates/vad/Cargo.toml index c29ace3691..a114c4a8ed 100644 --- a/crates/vad/Cargo.toml +++ b/crates/vad/Cargo.toml @@ -3,12 +3,16 @@ name = "vad" version = "0.1.0" edition = "2021" +[features] +default = [] +load-dynamic = ["ort/load-dynamic"] + [dependencies] serde = { workspace = true } thiserror = { workspace = true } ndarray = "0.16" -ort = "=2.0.0-rc.9" +ort = { version = "=2.0.0-rc.10", features = ["ndarray"] } [dev-dependencies] hypr-data = { workspace = true } diff --git a/crates/vad/src/lib.rs b/crates/vad/src/lib.rs index f1d84806a6..82b422b1f7 100644 --- a/crates/vad/src/lib.rs +++ b/crates/vad/src/lib.rs @@ -2,7 +2,10 @@ mod error; pub use error::*; use ndarray::{Array1, Array2, Array3, ArrayBase, Ix1, Ix3, OwnedRepr}; -use ort::session::{builder::GraphOptimizationLevel, Session}; +use ort::{ + session::{builder::GraphOptimizationLevel, Session}, + value::TensorRef, +}; const MODEL_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/assets/model.onnx")); @@ -46,30 +49,30 @@ impl Vad { let audio_tensor = Array2::from_shape_vec((1, samples), audio_chunk.to_vec())?; let mut result = self.session.run(ort::inputs![ - audio_tensor.view(), - self.sample_rate_tensor.view(), - self.h_tensor.view(), - self.c_tensor.view() - ]?)?; + TensorRef::from_array_view(audio_tensor.view())?, + TensorRef::from_array_view(self.sample_rate_tensor.view())?, + TensorRef::from_array_view(self.h_tensor.view())?, + TensorRef::from_array_view(self.c_tensor.view())?, + ])?; // Update internal state tensors self.h_tensor = result .get("hn") .ok_or(Error::InvalidOutput)? - .try_extract_tensor::()? + .try_extract_array::()? .to_owned() .into_shape_with_order((2, 1, 64))?; self.c_tensor = result .get("cn") .ok_or(Error::InvalidOutput)? - .try_extract_tensor::()? + .try_extract_array::()? .to_owned() .into_shape_with_order((2, 1, 64))?; let prob_tensor = result.remove("output").ok_or(Error::InvalidOutput)?; let prob = *prob_tensor - .try_extract_tensor::()? + .try_extract_array::()? .first() .ok_or(Error::InvalidOutput)?; diff --git a/plugins/listener/Cargo.toml b/plugins/listener/Cargo.toml index d56758c8ae..220e041b3d 100644 --- a/plugins/listener/Cargo.toml +++ b/plugins/listener/Cargo.toml @@ -60,3 +60,7 @@ statig = { workspace = true, features = ["async"] } [target."cfg(target_os = \"macos\")".dependencies] objc2 = { workspace = true } objc2-foundation = { workspace = true } + +[features] +default = [] +load-dynamic = ["hypr-aec/load-dynamic"] diff --git a/plugins/local-stt/Cargo.toml b/plugins/local-stt/Cargo.toml index 86983ba301..a0dacf0861 100644 --- a/plugins/local-stt/Cargo.toml +++ b/plugins/local-stt/Cargo.toml @@ -17,6 +17,7 @@ openblas = ["hypr-whisper-local/openblas"] metal = ["hypr-whisper-local/metal"] vulkan = ["hypr-whisper-local/vulkan"] openmp = ["hypr-whisper-local/openmp"] +load-dynamic = ["hypr-pyannote-local/load-dynamic", "hypr-chunker/load-dynamic"] [build-dependencies] tauri-plugin = { workspace = true, features = ["build"] }