diff --git a/Cargo.lock b/Cargo.lock index f4f0cd4d45..f86cdc3c68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "arboard" -version = "3.6.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f533f8e0af236ffe5eb979b99381df3258853f00ba2e44b6e1955292c75227" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" dependencies = [ "clipboard-win", "image 0.25.6", @@ -197,7 +197,7 @@ dependencies = [ "objc2-foundation 0.3.1", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "wl-clipboard-rs", "x11rb", ] @@ -281,9 +281,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb812ffb58524bdd10860d7d974e2f01cc0950c2438a74ee5ec2e2280c6c4ffa" +checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" dependencies = [ "async-task", "concurrent-queue", @@ -829,9 +829,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d07aa9a93b00c76f71bc35d598bed923f6d4f3a9ca5c24b7737ae1a292841c0" +checksum = "dd0b03af37dad7a14518b7691d81acb0f8222604ad3d1b02f6b4bed5188c0cd5" dependencies = [ "serde", ] @@ -928,6 +928,7 @@ dependencies = [ name = "cap-camera-mediafoundation" version = "0.1.0" dependencies = [ + "cap-mediafoundation-utils", "inquire", "thiserror 1.0.69", "tracing", @@ -941,6 +942,7 @@ version = "0.1.0" dependencies = [ "cap-camera-directshow", "cap-camera-mediafoundation", + "cap-mediafoundation-utils", "inquire", "thiserror 1.0.69", "windows 0.60.0", @@ -951,6 +953,7 @@ dependencies = [ name = "cap-cpal-ffmpeg" version = "0.1.0" dependencies = [ + "cap-ffmpeg-utils", "cpal 0.15.3 (git+https://github.com/RustAudio/cpal?rev=f43d36e55494993bbbde3299af0c53e5cdf4d4cf)", "ffmpeg-next", ] @@ -1093,14 +1096,63 @@ dependencies = [ "tracing", ] +[[package]] +name = "cap-enc-avfoundation" +version = "0.1.0" +dependencies = [ + "cap-ffmpeg-utils", + "cap-media-info", + "cidre 0.11.0", + "ffmpeg-next", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "cap-enc-ffmpeg" +version = "0.1.0" +dependencies = [ + "cap-media-info", + "ffmpeg-next", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "cap-enc-gif" +version = "0.1.0" +dependencies = [ + "gifski", + "imgref", + "rgb", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "cap-enc-mediafoundation" +version = "0.1.0" +dependencies = [ + "cap-media-info", + "clap", + "futures", + "scap-direct3d", + "scap-targets", + "thiserror 1.0.69", + "windows 0.60.0", + "windows-core 0.60.1", + "windows-numerics 0.2.0", +] + [[package]] name = "cap-export" version = "0.1.0" dependencies = [ "cap-editor", + "cap-enc-ffmpeg", + "cap-enc-gif", "cap-flags", "cap-media", - "cap-media-encoders", "cap-media-info", "cap-project", "cap-rendering", @@ -1108,9 +1160,12 @@ dependencies = [ "clap", "ffmpeg-next", "futures", + "gifski", "image 0.25.6", + "imgref", "inquire", "mp4", + "rgb", "serde", "serde_json", "specta", @@ -1127,6 +1182,13 @@ dependencies = [ "inventory", ] +[[package]] +name = "cap-ffmpeg-utils" +version = "0.1.0" +dependencies = [ + "ffmpeg-next", +] + [[package]] name = "cap-flags" version = "0.1.0" @@ -1146,79 +1208,44 @@ dependencies = [ name = "cap-media" version = "0.1.0" dependencies = [ - "axum", - "cap-audio", - "cap-camera", - "cap-camera-ffmpeg", - "cap-camera-windows", - "cap-cpal-ffmpeg", - "cap-fail", - "cap-flags", - "cap-media-encoders", "cap-media-info", - "cap-project", - "cidre 0.11.0", - "cocoa 0.26.1", - "core-foundation 0.10.1", - "core-graphics 0.24.0", - "cpal 0.15.3 (git+https://github.com/RustAudio/cpal?rev=f43d36e55494993bbbde3299af0c53e5cdf4d4cf)", "ffmpeg-next", - "flume", - "futures", - "gif", - "image 0.25.6", - "indexmap 2.10.0", "inquire", - "kameo", - "num-traits", - "objc", - "objc-foundation", - "objc2-foundation 0.2.2", - "ringbuf", - "scap", - "scap-direct3d", - "scap-ffmpeg", - "scap-screencapturekit", - "scap-targets", - "screencapturekit", - "serde", - "specta", - "sync_wrapper", - "tempfile", "thiserror 1.0.69", "tokio", - "tokio-util", - "tracing", "tracing-subscriber", - "windows 0.60.0", - "windows-capture", ] [[package]] -name = "cap-media-encoders" +name = "cap-media-info" version = "0.1.0" dependencies = [ - "cap-audio", - "cap-fail", - "cap-flags", - "cap-media-info", - "cap-project", - "cidre 0.11.0", + "cap-ffmpeg-utils", + "cpal 0.15.3 (git+https://github.com/RustAudio/cpal?rev=f43d36e55494993bbbde3299af0c53e5cdf4d4cf)", "ffmpeg-next", - "gifski", - "imgref", - "rgb", "thiserror 1.0.69", - "tracing", ] [[package]] -name = "cap-media-info" +name = "cap-mediafoundation-ffmpeg" version = "0.1.0" dependencies = [ - "cpal 0.15.3 (git+https://github.com/RustAudio/cpal?rev=f43d36e55494993bbbde3299af0c53e5cdf4d4cf)", + "cap-media-info", + "cap-mediafoundation-utils", "ffmpeg-next", + "flume", "thiserror 1.0.69", + "tracing", + "windows 0.60.0", + "windows-numerics 0.2.0", +] + +[[package]] +name = "cap-mediafoundation-utils" +version = "0.1.0" +dependencies = [ + "windows 0.60.0", + "windows-core 0.60.1", ] [[package]] @@ -1244,11 +1271,16 @@ dependencies = [ "cap-camera-ffmpeg", "cap-cursor-capture", "cap-cursor-info", + "cap-enc-avfoundation", + "cap-enc-ffmpeg", + "cap-enc-mediafoundation", "cap-fail", + "cap-ffmpeg-utils", "cap-flags", "cap-media", - "cap-media-encoders", "cap-media-info", + "cap-mediafoundation-ffmpeg", + "cap-mediafoundation-utils", "cap-project", "cap-utils", "chrono", @@ -1523,9 +1555,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.42" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" +checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" dependencies = [ "clap_builder", "clap_derive", @@ -1533,9 +1565,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.42" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" +checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" dependencies = [ "anstream", "anstyle", @@ -1545,9 +1577,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -3730,7 +3762,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2", "system-configuration", "tokio", "tower-service", @@ -6204,9 +6236,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -6215,7 +6247,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.5.10", + "socket2", "thiserror 2.0.12", "tokio", "tracing", @@ -6224,9 +6256,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", "getrandom 0.3.3", @@ -6245,16 +6277,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6956,6 +6988,7 @@ dependencies = [ name = "scap-ffmpeg" version = "0.1.0" dependencies = [ + "cap-ffmpeg-utils", "cidre 0.11.0", "cpal 0.15.3 (git+https://github.com/RustAudio/cpal?rev=f43d36e55494993bbbde3299af0c53e5cdf4d4cf)", "ffmpeg-next", @@ -7073,12 +7106,13 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "screencapturekit" -version = "0.3.5" -source = "git+https://github.com/CapSoftware/screencapturekit-rs?rev=7ff1e103742e56c8f6c2e940b5e52684ed0bed69#7ff1e103742e56c8f6c2e940b5e52684ed0bed69" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c62ae379564f834110c5b020cc731a7c14b8ec4879b4367579a467a0a704c21" dependencies = [ "block2 0.6.1", "core-foundation 0.10.1", - "core-graphics 0.24.0", + "core-graphics 0.25.0", "core-media-rs", "core-utils-rs", "core-video-rs", @@ -7630,16 +7664,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.0" @@ -8120,9 +8144,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tauri" -version = "2.8.2" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a54629607ea3084a8b455c1ebe888cbdfc4de02fa5edb2e40db0dc97091007e3" +checksum = "5d545ccf7b60dcd44e07c6fb5aeb09140966f0aabd5d2aa14a6821df7bc99348" dependencies = [ "anyhow", "bytes", @@ -8286,11 +8310,12 @@ dependencies = [ [[package]] name = "tauri-plugin-deep-link" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fec67f32d7a06d80bd3dc009fdb678c35a66116d9cb8cd2bb32e406c2b5bbd2" +checksum = "6d430110d4ee102a9b673d3c03ff48098c80fe8ca71ba1ff52d8a5919538a1a6" dependencies = [ "dunce", + "plist", "rust-ini", "serde", "serde_json", @@ -8385,13 +8410,13 @@ dependencies = [ [[package]] name = "tauri-plugin-notification" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe06ed89cff6d0ec06ff4f544fb961e4718348a33309f56ccb2086e77bc9116" +checksum = "d2fbc86b929b5376ab84b25c060f966d146b2fbd59b6af8264027b343c82c219" dependencies = [ "log", "notify-rust", - "rand 0.8.5", + "rand 0.9.2", "serde", "serde_json", "serde_repr", @@ -8483,9 +8508,9 @@ dependencies = [ [[package]] name = "tauri-plugin-shell" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b9ffadec5c3523f11e8273465cacb3d86ea7652a28e6e2a2e9b5c182f791d25" +checksum = "54777d0c0d8add34eea3ced84378619ef5b97996bd967d3038c668feefd21071" dependencies = [ "encoding_rs", "log", @@ -8504,9 +8529,9 @@ dependencies = [ [[package]] name = "tauri-plugin-single-instance" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50a0e5a4ce43cb3a733c3aef85e8478bc769dac743c615e26639cbf5d953faf7" +checksum = "236043404a4d1502ed7cce11a8ec88ea1e85597eec9887b4701bb10b66b13b6e" dependencies = [ "serde", "serde_json", @@ -8608,9 +8633,9 @@ dependencies = [ [[package]] name = "tauri-runtime-wry" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb0f10f831f75832ac74d14d98f701868f9a8adccef2c249b466cf70b607db9" +checksum = "c1fe9d48bd122ff002064e88cfcd7027090d789c4302714e68fcccba0f4b7807" dependencies = [ "gtk", "http", @@ -8919,7 +8944,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2 0.6.0", + "socket2", "tokio-macros", "tracing", "windows-sys 0.59.0", @@ -10879,9 +10904,9 @@ checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "wry" -version = "0.53.1" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5698e50a589268aec06d2219f48b143222f7b5ad9aa690118b8dce0a8dcac574" +checksum = "31f0e9642a0d061f6236c54ccae64c2722a7879ad4ec7dff59bd376d446d8e90" dependencies = [ "base64 0.22.1", "block2 0.6.1", @@ -11051,9 +11076,9 @@ dependencies = [ [[package]] name = "zbus" -version = "5.9.0" +version = "5.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb4f9a464286d42851d18a605f7193b8febaf5b0919d71c6399b7b26e5b0aad" +checksum = "67a073be99ace1adc48af593701c8015cd9817df372e14a1a6b0ee8f8bf043be" dependencies = [ "async-broadcast", "async-executor", @@ -11076,7 +11101,7 @@ dependencies = [ "tokio", "tracing", "uds_windows", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "winnow 0.7.12", "zbus_macros", "zbus_names", @@ -11085,9 +11110,9 @@ dependencies = [ [[package]] name = "zbus_macros" -version = "5.9.0" +version = "5.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef9859f68ee0c4ee2e8cde84737c78e3f4c54f946f2a38645d0d4c7a95327659" +checksum = "0e80cd713a45a49859dcb648053f63265f4f2851b6420d47a958e5697c68b131" dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", @@ -11198,9 +11223,9 @@ dependencies = [ [[package]] name = "zip" -version = "4.3.0" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aed4ac33e8eb078c89e6cbb1d5c4c7703ec6d299fc3e7c3695af8f8b423468b" +checksum = "8835eb39822904d39cb19465de1159e05d371973f0c6df3a365ad50565ddc8b9" dependencies = [ "arbitrary", "crc32fast", @@ -11240,9 +11265,9 @@ dependencies = [ [[package]] name = "zvariant" -version = "5.6.0" +version = "5.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91b3680bb339216abd84714172b5138a4edac677e641ef17e1d8cb1b3ca6e6f" +checksum = "999dd3be73c52b1fccd109a4a81e4fcd20fab1d3599c8121b38d04e1419498db" dependencies = [ "endi", "enumflags2", @@ -11255,9 +11280,9 @@ dependencies = [ [[package]] name = "zvariant_derive" -version = "5.6.0" +version = "5.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8c68501be459a8dbfffbe5d792acdd23b4959940fc87785fb013b32edbc208" +checksum = "6643fd0b26a46d226bd90d3f07c1b5321fe9bb7f04673cb37ac6d6883885b68e" dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", @@ -11268,14 +11293,13 @@ dependencies = [ [[package]] name = "zvariant_utils" -version = "3.2.0" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16edfee43e5d7b553b77872d99bc36afdda75c223ca7ad5e3fbecd82ca5fc34" +checksum = "c6949d142f89f6916deca2232cf26a8afacf2b9fdc35ce766105e104478be599" dependencies = [ "proc-macro2", "quote", "serde", - "static_assertions", "syn 2.0.104", "winnow 0.7.12", ] diff --git a/Cargo.toml b/Cargo.toml index 87385e6a63..46cd816eaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,48 +9,48 @@ anyhow = { version = "1.0.86" } cpal = { git = "https://github.com/RustAudio/cpal", rev = "f43d36e55494993bbbde3299af0c53e5cdf4d4cf" } ffmpeg = { package = "ffmpeg-next", git = "https://github.com/CapSoftware/rust-ffmpeg", rev = "49db1fede112" } tokio = { version = "1.39.3", features = [ - "macros", - "process", - "fs", - "sync", - "rt", - "rt-multi-thread", - "time", + "macros", + "process", + "fs", + "sync", + "rt", + "rt-multi-thread", + "time", ] } tauri = { version = "2.5.0", features = ["specta"] } specta = { version = "=2.0.0-rc.20", features = [ - "derive", - "serde_json", - "uuid", + "derive", + "serde_json", + "uuid", ] } serde = { version = "1", features = ["derive"] } scap = { git = "https://github.com/CapSoftware/scap", rev = "3cefe71561ff" } nokhwa = { git = "https://github.com/CapSoftware/nokhwa", rev = "b9c8079e82e2", features = [ - "input-native", - "serialize", + "input-native", + "serialize", ] } nokhwa-bindings-macos = { git = "https://github.com/CapSoftware/nokhwa", rev = "b9c8079e82e2" } wgpu = "25.0.0" flume = "0.11.0" thiserror = "1.0" sentry = { version = "0.34.0", features = [ - "anyhow", - "backtrace", - "debug-images", + "anyhow", + "backtrace", + "debug-images", ] } tracing = "0.1.41" futures = "0.3.31" cidre = { git = "https://github.com/CapSoftware/cidre", rev = "bf84b67079a8", features = [ - "macos_13_0", - "cv", - "cf", - "sc", - "av", - "blocks", - "async", - "dispatch", + "macos_13_0", + "cv", + "cf", + "sc", + "av", + "blocks", + "async", + "dispatch", ], default-features = false } windows = "0.60.0" @@ -77,7 +77,7 @@ opt-level = "s" # Optimize for binary size debug = true [patch.crates-io] -screencapturekit = { git = "https://github.com/CapSoftware/screencapturekit-rs", rev = "7ff1e103742e56c8f6c2e940b5e52684ed0bed69" } # branch = "cap-main" +# screencapturekit = { git = "https://github.com/CapSoftware/screencapturekit-rs", rev = "7ff1e103742e56c8f6c2e940b5e52684ed0bed69" } # branch = "cap-main" cidre = { git = "https://github.com/CapSoftware/cidre", rev = "bf84b67079a8" } # https://github.com/gfx-rs/wgpu/pull/7550 # wgpu = { git = "https://github.com/gfx-rs/wgpu", rev = "cd41a6e32a6239b65d1cecbeccde6a43a100914a" } diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 8310f29525..47489b27db 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -57,7 +57,7 @@ mp4 = "0.14.0" futures-intrusive = "0.5.0" anyhow.workspace = true futures = { workspace = true } -axum = { version = "0.7.5", features = ["ws"] } +axum = { version = "0.7.5", features = ["ws", "macros"] } tracing.workspace = true tempfile = "3.9.0" ffmpeg.workspace = true diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index e7a69027fa..060bd494b6 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2235,17 +2235,17 @@ pub async fn run(recording_logging_handle: LoggingHandle) { }) .build(tauri_context) .expect("error while running tauri application") - .run(move |handle, event| match event { + .run(move |_handle, event| match event { #[cfg(target_os = "macos")] tauri::RunEvent::Reopen { .. } => { - let has_window = handle.webview_windows().iter().any(|(label, _)| { + let has_window = _handle.webview_windows().iter().any(|(label, _)| { label.starts_with("editor-") || label.as_str() == "settings" || label.as_str() == "signin" }); if has_window { - if let Some(window) = handle + if let Some(window) = _handle .webview_windows() .iter() .find(|(label, _)| { @@ -2258,7 +2258,7 @@ pub async fn run(recording_logging_handle: LoggingHandle) { window.set_focus().ok(); } } else { - let handle = handle.clone(); + let handle = _handle.clone(); tokio::spawn(async move { let _ = ShowCapWindow::Main.show(&handle).await; }); diff --git a/apps/web/app/(org)/dashboard/caps/components/Folders.tsx b/apps/web/app/(org)/dashboard/caps/components/Folders.tsx index 17955be2e4..4c10c09440 100644 --- a/apps/web/app/(org)/dashboard/caps/components/Folders.tsx +++ b/apps/web/app/(org)/dashboard/caps/components/Folders.tsx @@ -1,6 +1,6 @@ "use client"; -import { Fit, Layout, RiveFile, useRive } from "@rive-app/react-canvas"; +import { Fit, Layout, type RiveFile, useRive } from "@rive-app/react-canvas"; import React, { useImperativeHandle } from "react"; import { useTheme } from "../../Contexts"; diff --git a/crates/camera-avfoundation/examples/cli.rs b/crates/camera-avfoundation/examples/cli.rs index 95014b5a22..102615dd7d 100644 --- a/crates/camera-avfoundation/examples/cli.rs +++ b/crates/camera-avfoundation/examples/cli.rs @@ -1,194 +1,200 @@ -#![cfg(target_os = "macos")] - -use cap_camera_avfoundation::{ - CallbackOutputDelegate, CallbackOutputDelegateInner, YCbCrMatrix, list_video_devices, -}; -use cidre::*; -use clap::{Parser, Subcommand}; -use inquire::Select; -use std::{fmt::Display, ops::Deref}; - -#[derive(Parser)] -struct Cli { - #[command(subcommand)] - command: Commands, +fn main() { + #[cfg(target_os = "macos")] + macos::main(); } -#[derive(Subcommand)] -enum Commands { - /// Print details of a device - Device, -} +#[cfg(target_os = "macos")] +mod macos { + use cap_camera_avfoundation::{ + CallbackOutputDelegate, CallbackOutputDelegateInner, YCbCrMatrix, list_video_devices, + }; + use cidre::*; + use clap::{Parser, Subcommand}; + use inquire::Select; + use std::{fmt::Display, ops::Deref}; + + #[derive(Parser)] + struct Cli { + #[command(subcommand)] + command: Commands, + } -pub fn main() { - let _devices = list_video_devices(); - let devices = _devices - .iter() - .enumerate() - .map(|(i, v)| CaptureDeviceSelectOption(v, i)) - .collect::>(); - - let selected = Select::new("Select a device", devices).prompt().unwrap(); - let mut selected_device = _devices.get(selected.1).unwrap(); - - println!("Info for device '{}'", selected_device.localized_name()); - - let formats = selected_device.formats(); - - let mut _formats = vec![]; - - for (i, format) in formats.iter().enumerate() { - let desc = format.format_desc(); - - let color_space = desc - .ext(cm::FormatDescExtKey::ycbcr_matrix()) - .map(|v| { - v.try_as_string() - .and_then(|v| YCbCrMatrix::try_from(v).ok()) - }) - .unwrap_or(Some(YCbCrMatrix::Rec601)); - - let fr_ranges = format.video_supported_frame_rate_ranges(); - - for fr_range in fr_ranges.iter() { - _formats.push(Format { - index: i, - width: desc.dims().width, - height: desc.dims().height, - fourcc: desc.media_sub_type(), - color_space, - max_frame_rate: ( - fr_range.min_frame_duration().value, - fr_range.min_frame_duration().scale, - ), - }); - } + #[derive(Subcommand)] + enum Commands { + /// Print details of a device + Device, } - let selected_format = if _formats.len() > 1 { - inquire::Select::new("Select a format", _formats) - .prompt() - .unwrap() - } else { - _formats.remove(0) - }; + pub fn main() { + let _devices = list_video_devices(); + let devices = _devices + .iter() + .enumerate() + .map(|(i, v)| CaptureDeviceSelectOption(v, i)) + .collect::>(); + + let selected = Select::new("Select a device", devices).prompt().unwrap(); + let mut selected_device = _devices.get(selected.1).unwrap(); + + println!("Info for device '{}'", selected_device.localized_name()); + + let formats = selected_device.formats(); + + let mut _formats = vec![]; + + for (i, format) in formats.iter().enumerate() { + let desc = format.format_desc(); + + let color_space = desc + .ext(cm::FormatDescExtKey::ycbcr_matrix()) + .map(|v| { + v.try_as_string() + .and_then(|v| YCbCrMatrix::try_from(v).ok()) + }) + .unwrap_or(Some(YCbCrMatrix::Rec601)); + + let fr_ranges = format.video_supported_frame_rate_ranges(); + + for fr_range in fr_ranges.iter() { + _formats.push(Format { + index: i, + width: desc.dims().width, + height: desc.dims().height, + fourcc: desc.media_sub_type(), + color_space, + max_frame_rate: ( + fr_range.min_frame_duration().value, + fr_range.min_frame_duration().scale, + ), + }); + } + } - let input = av::capture::DeviceInput::with_device(&selected_device).unwrap(); - let queue = dispatch::Queue::new(); - let delegate = - CallbackOutputDelegate::with(CallbackOutputDelegateInner::new(Box::new(|data| { - let Some(image_buf) = data.sample_buf.image_buf() else { - return; - }; - - let total_bytes = if image_buf.plane_count() > 0 { - (0..image_buf.plane_count()) - .map(|i| image_buf.plane_bytes_per_row(i) * image_buf.plane_height(i)) - .sum::() + let selected_format = if _formats.len() > 1 { + inquire::Select::new("Select a format", _formats) + .prompt() + .unwrap() + } else { + _formats.remove(0) + }; + + let input = av::capture::DeviceInput::with_device(&selected_device).unwrap(); + let queue = dispatch::Queue::new(); + let delegate = + CallbackOutputDelegate::with(CallbackOutputDelegateInner::new(Box::new(|data| { + let Some(image_buf) = data.sample_buf.image_buf() else { + return; + }; + + let total_bytes = if image_buf.plane_count() > 0 { + (0..image_buf.plane_count()) + .map(|i| image_buf.plane_bytes_per_row(i) * image_buf.plane_height(i)) + .sum::() + } else { + image_buf.plane_bytes_per_row(0) * image_buf.plane_height(0) + }; + + let mut format = image_buf.pixel_format().0.to_be_bytes(); + let format_fourcc = four_cc_to_str(&mut format); + + println!( + "New frame: {}x{}, {:.2}pts, {total_bytes} bytes, format={format_fourcc}", + image_buf.width(), + image_buf.height(), + data.sample_buf.pts().value as f64 / data.sample_buf.pts().scale as f64, + ) + }))); + + let mut output = av::capture::VideoDataOutput::new(); + + let mut session = av::capture::Session::new(); + + session.configure(|s| { + if s.can_add_input(&input) { + s.add_input(&input); } else { - image_buf.plane_bytes_per_row(0) * image_buf.plane_height(0) - }; + panic!("can't add input"); + } - let mut format = image_buf.pixel_format().0.to_be_bytes(); - let format_fourcc = four_cc_to_str(&mut format); + s.add_output(&output); - println!( - "New frame: {}x{}, {:.2}pts, {total_bytes} bytes, format={format_fourcc}", - image_buf.width(), - image_buf.height(), - data.sample_buf.pts().value as f64 / data.sample_buf.pts().scale as f64, - ) - }))); - - let mut output = av::capture::VideoDataOutput::new(); + let mut _lock = selected_device.config_lock().unwrap(); - let mut session = av::capture::Session::new(); + _lock.set_active_format(&formats[selected_format.index]); + }); - session.configure(|s| { - if s.can_add_input(&input) { - s.add_input(&input); - } else { - panic!("can't add input"); - } + output.set_sample_buf_delegate(Some(delegate.as_ref()), Some(&queue)); - s.add_output(&output); + let video_settings = ns::Dictionary::with_keys_values( + &[cv::pixel_buffer_keys::pixel_format().as_ns()], + &[ns::Number::with_u32(selected_format.fourcc).as_id_ref()], + ); + output + .set_video_settings(Some(video_settings.as_ref())) + .unwrap(); - let mut _lock = selected_device.config_lock().unwrap(); + // The device config must stay locked while running starts, + // otherwise start_running can overwrite the active format on macOS + // https://stackoverflow.com/questions/36689578/avfoundation-capturing-video-with-custom-resolution + { + let mut _lock = selected_device.config_lock().unwrap(); - _lock.set_active_format(&formats[selected_format.index]); - }); + _lock.set_active_format(&formats[selected_format.index]); - output.set_sample_buf_delegate(Some(delegate.as_ref()), Some(&queue)); - - let video_settings = ns::Dictionary::with_keys_values( - &[cv::pixel_buffer_keys::pixel_format().as_ns()], - &[ns::Number::with_u32(selected_format.fourcc).as_id_ref()], - ); - output - .set_video_settings(Some(video_settings.as_ref())) - .unwrap(); + session.start_running(); + } - // The device config must stay locked while running starts, - // otherwise start_running can overwrite the active format on macOS - // https://stackoverflow.com/questions/36689578/avfoundation-capturing-video-with-custom-resolution - { - let mut _lock = selected_device.config_lock().unwrap(); + std::thread::sleep(std::time::Duration::from_secs(10)); - _lock.set_active_format(&formats[selected_format.index]); + session.stop_running(); - session.start_running(); + std::thread::sleep(std::time::Duration::from_secs(10)); } - std::thread::sleep(std::time::Duration::from_secs(10)); - - session.stop_running(); - - std::thread::sleep(std::time::Duration::from_secs(10)); -} - -struct Format { - index: usize, - width: i32, - height: i32, - fourcc: FourCharCode, - #[allow(unused)] - color_space: Option, - max_frame_rate: (i64, i32), -} + struct Format { + index: usize, + width: i32, + height: i32, + fourcc: FourCharCode, + #[allow(unused)] + color_space: Option, + max_frame_rate: (i64, i32), + } -impl Display for Format { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}x{}, {} max fps ({}/{}) {}", - self.width, - self.height, - self.max_frame_rate.1 as f32 / self.max_frame_rate.0 as f32, - self.max_frame_rate.0, - self.max_frame_rate.1, - four_cc_to_string(self.fourcc.to_be_bytes()) - ) + impl Display for Format { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}x{}, {} max fps ({}/{}) {}", + self.width, + self.height, + self.max_frame_rate.1 as f32 / self.max_frame_rate.0 as f32, + self.max_frame_rate.0, + self.max_frame_rate.1, + four_cc_to_string(self.fourcc.to_be_bytes()) + ) + } } -} -struct CaptureDeviceSelectOption<'a>(&'a av::CaptureDevice, usize); + struct CaptureDeviceSelectOption<'a>(&'a av::CaptureDevice, usize); -impl<'a> Display for CaptureDeviceSelectOption<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} ({})", self.0.localized_name(), self.0.unique_id()) + impl<'a> Display for CaptureDeviceSelectOption<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ({})", self.0.localized_name(), self.0.unique_id()) + } } -} -impl AsRef for CaptureDeviceSelectOption<'_> { - fn as_ref(&self) -> &av::CaptureDevice { - self.0 + impl AsRef for CaptureDeviceSelectOption<'_> { + fn as_ref(&self) -> &av::CaptureDevice { + self.0 + } } -} -impl Deref for CaptureDeviceSelectOption<'_> { - type Target = av::CaptureDevice; + impl Deref for CaptureDeviceSelectOption<'_> { + type Target = av::CaptureDevice; - fn deref(&self) -> &Self::Target { - self.0 + fn deref(&self) -> &Self::Target { + self.0 + } } } diff --git a/crates/camera-mediafoundation/Cargo.toml b/crates/camera-mediafoundation/Cargo.toml index 3bc52bb58b..db4970a4fb 100644 --- a/crates/camera-mediafoundation/Cargo.toml +++ b/crates/camera-mediafoundation/Cargo.toml @@ -9,11 +9,12 @@ tracing.workspace = true thiserror.workspace = true [target.'cfg(windows)'.dependencies] +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } windows-core = { workspace = true } windows = { workspace = true, features = [ - "Win32_Media_MediaFoundation", - "Win32_Media_DirectShow", - "Win32_System_Com", + "Win32_Media_MediaFoundation", + "Win32_Media_DirectShow", + "Win32_System_Com", ] } [dev-dependencies] diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index 349df50610..175120a5c5 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -1,13 +1,13 @@ #![cfg(windows)] #![allow(non_snake_case)] +use cap_mediafoundation_utils::*; use std::{ ffi::OsString, fmt::Display, mem::MaybeUninit, ops::{Deref, DerefMut}, os::windows::ffi::OsStringExt, - ptr::null_mut, slice::from_raw_parts, sync::{ Mutex, @@ -15,7 +15,6 @@ use std::{ }, time::{Duration, Instant}, }; - use tracing::error; use windows::Win32::{ Foundation::{S_FALSE, *}, @@ -541,51 +540,6 @@ fn get_device_model_id(device_id: &str) -> Option { Some(format!("{id_vendor}:{id_product}")) } -pub trait IMFMediaBufferExt { - fn lock(&self) -> windows_core::Result>; -} - -impl IMFMediaBufferExt for IMFMediaBuffer { - fn lock(&self) -> windows_core::Result> { - let mut bytes_ptr = null_mut(); - let mut size = 0; - - unsafe { - self.Lock(&mut bytes_ptr, None, Some(&mut size))?; - } - - Ok(IMFMediaBufferLock { - source: self, - bytes: unsafe { std::slice::from_raw_parts_mut(bytes_ptr, size as usize) }, - }) - } -} - -pub struct IMFMediaBufferLock<'a> { - source: &'a IMFMediaBuffer, - bytes: &'a mut [u8], -} - -impl<'a> Drop for IMFMediaBufferLock<'a> { - fn drop(&mut self) { - let _ = unsafe { self.source.Unlock() }; - } -} - -impl<'a> Deref for IMFMediaBufferLock<'a> { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - self.bytes - } -} - -impl<'a> DerefMut for IMFMediaBufferLock<'a> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.bytes - } -} - pub struct CallbackData { pub sample: IMFSample, pub reference_time: Instant, diff --git a/crates/camera-windows/Cargo.toml b/crates/camera-windows/Cargo.toml index 4960cfcab4..5f3378c321 100644 --- a/crates/camera-windows/Cargo.toml +++ b/crates/camera-windows/Cargo.toml @@ -9,6 +9,7 @@ thiserror.workspace = true inquire = "0.7.5" [target.'cfg(windows)'.dependencies] +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } cap-camera-mediafoundation = { path = "../camera-mediafoundation" } cap-camera-directshow = { path = "../camera-directshow" } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index 773555ac9e..9214e10978 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -1,7 +1,7 @@ #![cfg(windows)] use cap_camera_directshow::{AM_MEDIA_TYPEVideoExt, AMMediaType}; -use cap_camera_mediafoundation::{IMFMediaBufferExt, IMFMediaBufferLock}; +use cap_mediafoundation_utils::*; use std::{ ffi::{OsStr, OsString}, fmt::{Debug, Display}, diff --git a/crates/cpal-ffmpeg/Cargo.toml b/crates/cpal-ffmpeg/Cargo.toml index 4ea431bd53..f0931bd38c 100644 --- a/crates/cpal-ffmpeg/Cargo.toml +++ b/crates/cpal-ffmpeg/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] cpal.workspace = true ffmpeg.workspace = true +cap-ffmpeg-utils = { path = "../ffmpeg-utils" } [lints] workspace = true diff --git a/crates/cpal-ffmpeg/src/lib.rs b/crates/cpal-ffmpeg/src/lib.rs index 2b0a46eef2..80f90a2325 100644 --- a/crates/cpal-ffmpeg/src/lib.rs +++ b/crates/cpal-ffmpeg/src/lib.rs @@ -1,3 +1,4 @@ +use cap_ffmpeg_utils::PlanarData; use cpal::{SampleFormat, StreamConfig}; use ffmpeg::format::{Sample, sample}; @@ -43,39 +44,3 @@ impl DataExt for ::cpal::Data { ffmpeg_frame } } - -pub trait PlanarData { - fn plane_data(&self, index: usize) -> &[u8]; - - fn plane_data_mut(&mut self, index: usize) -> &mut [u8]; -} - -impl PlanarData for ffmpeg::frame::Audio { - #[inline] - fn plane_data(&self, index: usize) -> &[u8] { - if index >= self.planes() { - panic!("out of bounds"); - } - - unsafe { - std::slice::from_raw_parts( - (*self.as_ptr()).data[index], - (*self.as_ptr()).linesize[0] as usize, - ) - } - } - - #[inline] - fn plane_data_mut(&mut self, index: usize) -> &mut [u8] { - if index >= self.planes() { - panic!("out of bounds"); - } - - unsafe { - std::slice::from_raw_parts_mut( - (*self.as_mut_ptr()).data[index], - (*self.as_ptr()).linesize[0] as usize, - ) - } - } -} diff --git a/crates/cursor-capture/src/position.rs b/crates/cursor-capture/src/position.rs index 40312dcca3..8ff1c12c07 100644 --- a/crates/cursor-capture/src/position.rs +++ b/crates/cursor-capture/src/position.rs @@ -205,151 +205,159 @@ impl std::fmt::Debug for NormalizedCursorPosition { } } -#[cfg(test)] -mod tests { - use super::*; - use scap_targets::Display; - - // Helper function to create a mock Display for testing - fn mock_display() -> Display { - Display::list()[0] - } - - #[test] - fn test_with_crop_no_change() { - let display = mock_display(); - let original_normalized = NormalizedCursorPosition { - x: 0.5, - y: 0.5, - crop_position: LogicalPosition::new(0.0, 0.0), - crop_size: LogicalSize::new(1.0, 1.0), - display, - }; - - let cropped_position = LogicalPosition::new(0.0, 0.0); - let cropped_size = LogicalSize::new(1.0, 1.0); - let new_normalized = original_normalized.with_crop(cropped_position, cropped_size); - - assert_eq!(new_normalized.x, 0.5); - assert_eq!(new_normalized.y, 0.5); - assert_eq!( - new_normalized.crop_position(), - LogicalPosition::new(0.0, 0.0) - ); - assert_eq!(new_normalized.crop_size(), LogicalSize::new(1.0, 1.0)); - } - - #[test] - fn test_with_crop_centered() { - let display = mock_display(); - let original_normalized = NormalizedCursorPosition { - x: 0.5, - y: 0.5, - crop_position: LogicalPosition::new(0.0, 0.0), - crop_size: LogicalSize::new(1.0, 1.0), - display, - }; - - let cropped_position = LogicalPosition::new(0.25, 0.25); - let cropped_size = LogicalSize::new(0.5, 0.5); - let new_normalized = original_normalized.with_crop(cropped_position, cropped_size); - - // Original point (0.5, 0.5) is in the center of the (0,0) to (1,1) range. - // The new crop is from (0.25, 0.25) to (0.75, 0.75). - // The original point (0.5, 0.5) should still be in the center of this new crop. - let expected_x = (0.5 * 1.0 + 0.0 - 0.25) / 0.5; - let expected_y = (0.5 * 1.0 + 0.0 - 0.25) / 0.5; - - assert!((new_normalized.x - expected_x).abs() < f64::EPSILON); - assert!((new_normalized.y - expected_y).abs() < f64::EPSILON); - assert_eq!(new_normalized.crop_position(), cropped_position); - assert_eq!(new_normalized.crop_size(), cropped_size); - } - - #[test] - fn test_with_crop_top_left_of_crop() { - let display = mock_display(); - - let cropped_position = LogicalPosition::new(0.25, 0.25); - let cropped_size = LogicalSize::new(0.5, 0.5); - - let original_normalized_at_crop_tl = NormalizedCursorPosition { - x: 0.25, - y: 0.25, - crop_position: LogicalPosition::new(0.0, 0.0), - crop_size: LogicalSize::new(1.0, 1.0), - display, - }; - - let new_normalized = - original_normalized_at_crop_tl.with_crop(cropped_position, cropped_size); - - // The point that was at the top-left of the crop in the original space - // should now be at (0.0, 0.0) in the new cropped space. - assert!((new_normalized.x - 0.0).abs() < f64::EPSILON); - assert!((new_normalized.y - 0.0).abs() < f64::EPSILON); - assert_eq!(new_normalized.crop_position(), cropped_position); - assert_eq!(new_normalized.crop_size(), cropped_size); - } - - #[test] - fn test_with_crop_bottom_right_of_crop() { - let display = mock_display(); - - let cropped_position = LogicalPosition::new(0.25, 0.25); - let cropped_size = LogicalSize::new(0.5, 0.5); - - let original_normalized_at_crop_br = NormalizedCursorPosition { - x: 0.75, - y: 0.75, - crop_position: LogicalPosition::new(0.0, 0.0), - crop_size: LogicalSize::new(1.0, 1.0), - display, - }; - - let new_normalized = - original_normalized_at_crop_br.with_crop(cropped_position, cropped_size); - - // The point that was at the bottom-right of the crop in the original space - // should now be at (1.0, 1.0) in the new cropped space. - assert!((new_normalized.x - 1.0).abs() < f64::EPSILON); - assert!((new_normalized.y - 1.0).abs() < f64::EPSILON); - assert_eq!(new_normalized.crop_position(), cropped_position); - assert_eq!(new_normalized.crop_size(), cropped_size); - } - - #[test] - fn test_with_crop_from_existing_crop() { - let display = mock_display(); - let original_normalized = NormalizedCursorPosition { - x: 0.5, // This 0.5 is within the first crop - y: 0.5, // This 0.5 is within the first crop - crop_position: LogicalPosition::new(0.1, 0.1), - crop_size: LogicalSize::new(0.8, 0.8), - display, - }; - - // The raw position of the cursor is 0.5 within the 0.1 to 0.9 range. - // Raw x = 0.5 * 0.8 + 0.1 = 0.4 + 0.1 = 0.5 - // Raw y = 0.5 * 0.8 + 0.1 = 0.4 + 0.1 = 0.5 - - let second_crop_position = LogicalPosition::new(0.2, 0.2); - let second_crop_size = LogicalSize::new(0.6, 0.6); - - // The second crop is from 0.2 to 0.8 in the original space. - // The raw position is (0.5, 0.5). - // In the second crop space, this should be: - // x = (0.5 - 0.2) / 0.6 = 0.3 / 0.6 = 0.5 - // y = (0.5 - 0.2) / 0.6 = 0.3 / 0.6 = 0.5 - - let new_normalized = original_normalized.with_crop(second_crop_position, second_crop_size); - - assert!((new_normalized.x - 0.5).abs() < f64::EPSILON); - assert!((new_normalized.y - 0.5).abs() < f64::EPSILON); - assert_eq!( - new_normalized.crop_position(), - LogicalPosition::new(0.2, 0.2) - ); - assert_eq!(new_normalized.crop_size(), LogicalSize::new(0.6, 0.6)); - } -} +// #[cfg(test)] +// mod tests { +// use super::*; +// use scap_targets::Display; + +// // Helper function to create a mock Display for testing +// fn mock_display() -> Display { +// Display::list()[0] +// } + +// #[test] +// fn test_with_crop_no_change() { +// let display = mock_display(); +// let original_normalized = NormalizedCursorPosition { +// x: 0.5, +// y: 0.5, +// crop: CursorCropBounds { +// x: 0.0, +// y: 0.0, +// width: 1.0, +// height: 1.0, +// }, +// display, +// }; + +// let crop = CursorCropBounds { +// x: 0.0, +// y: 0.0, +// width: 1.0, +// height: 1.0, +// }; +// let new_normalized = original_normalized.with_crop(crop); + +// assert_eq!(new_normalized.x, 0.5); +// assert_eq!(new_normalized.y, 0.5); +// assert_eq!( +// new_normalized.crop_position(), +// LogicalPosition::new(0.0, 0.0) +// ); +// assert_eq!(new_normalized.crop_size(), LogicalSize::new(1.0, 1.0)); +// } + +// #[test] +// fn test_with_crop_centered() { +// let display = mock_display(); +// let original_normalized = NormalizedCursorPosition { +// x: 0.5, +// y: 0.5, +// crop_position: LogicalPosition::new(0.0, 0.0), +// crop_size: LogicalSize::new(1.0, 1.0), +// display, +// }; + +// let cropped_position = LogicalPosition::new(0.25, 0.25); +// let cropped_size = LogicalSize::new(0.5, 0.5); +// let new_normalized = original_normalized.with_crop(cropped_position, cropped_size); + +// // Original point (0.5, 0.5) is in the center of the (0,0) to (1,1) range. +// // The new crop is from (0.25, 0.25) to (0.75, 0.75). +// // The original point (0.5, 0.5) should still be in the center of this new crop. +// let expected_x = (0.5 * 1.0 + 0.0 - 0.25) / 0.5; +// let expected_y = (0.5 * 1.0 + 0.0 - 0.25) / 0.5; + +// assert!((new_normalized.x - expected_x).abs() < f64::EPSILON); +// assert!((new_normalized.y - expected_y).abs() < f64::EPSILON); +// assert_eq!(new_normalized.crop_position(), cropped_position); +// assert_eq!(new_normalized.crop_size(), cropped_size); +// } + +// #[test] +// fn test_with_crop_top_left_of_crop() { +// let display = mock_display(); + +// let cropped_position = LogicalPosition::new(0.25, 0.25); +// let cropped_size = LogicalSize::new(0.5, 0.5); + +// let original_normalized_at_crop_tl = NormalizedCursorPosition { +// x: 0.25, +// y: 0.25, +// crop_position: LogicalPosition::new(0.0, 0.0), +// crop_size: LogicalSize::new(1.0, 1.0), +// display, +// }; + +// let new_normalized = +// original_normalized_at_crop_tl.with_crop(cropped_position, cropped_size); + +// // The point that was at the top-left of the crop in the original space +// // should now be at (0.0, 0.0) in the new cropped space. +// assert!((new_normalized.x - 0.0).abs() < f64::EPSILON); +// assert!((new_normalized.y - 0.0).abs() < f64::EPSILON); +// assert_eq!(new_normalized.crop_position(), cropped_position); +// assert_eq!(new_normalized.crop_size(), cropped_size); +// } + +// #[test] +// fn test_with_crop_bottom_right_of_crop() { +// let display = mock_display(); + +// let cropped_position = LogicalPosition::new(0.25, 0.25); +// let cropped_size = LogicalSize::new(0.5, 0.5); + +// let original_normalized_at_crop_br = NormalizedCursorPosition { +// x: 0.75, +// y: 0.75, +// crop_position: LogicalPosition::new(0.0, 0.0), +// crop_size: LogicalSize::new(1.0, 1.0), +// display, +// }; + +// let new_normalized = +// original_normalized_at_crop_br.with_crop(cropped_position, cropped_size); + +// // The point that was at the bottom-right of the crop in the original space +// // should now be at (1.0, 1.0) in the new cropped space. +// assert!((new_normalized.x - 1.0).abs() < f64::EPSILON); +// assert!((new_normalized.y - 1.0).abs() < f64::EPSILON); +// assert_eq!(new_normalized.crop_position(), cropped_position); +// assert_eq!(new_normalized.crop_size(), cropped_size); +// } + +// #[test] +// fn test_with_crop_from_existing_crop() { +// let display = mock_display(); +// let original_normalized = NormalizedCursorPosition { +// x: 0.5, // This 0.5 is within the first crop +// y: 0.5, // This 0.5 is within the first crop +// crop_position: LogicalPosition::new(0.1, 0.1), +// crop_size: LogicalSize::new(0.8, 0.8), +// display, +// }; + +// // The raw position of the cursor is 0.5 within the 0.1 to 0.9 range. +// // Raw x = 0.5 * 0.8 + 0.1 = 0.4 + 0.1 = 0.5 +// // Raw y = 0.5 * 0.8 + 0.1 = 0.4 + 0.1 = 0.5 + +// let second_crop_position = LogicalPosition::new(0.2, 0.2); +// let second_crop_size = LogicalSize::new(0.6, 0.6); + +// // The second crop is from 0.2 to 0.8 in the original space. +// // The raw position is (0.5, 0.5). +// // In the second crop space, this should be: +// // x = (0.5 - 0.2) / 0.6 = 0.3 / 0.6 = 0.5 +// // y = (0.5 - 0.2) / 0.6 = 0.3 / 0.6 = 0.5 + +// let new_normalized = original_normalized.with_crop(second_crop_position, second_crop_size); + +// assert!((new_normalized.x - 0.5).abs() < f64::EPSILON); +// assert!((new_normalized.y - 0.5).abs() < f64::EPSILON); +// assert_eq!( +// new_normalized.crop_position(), +// LogicalPosition::new(0.2, 0.2) +// ); +// assert_eq!(new_normalized.crop_size(), LogicalSize::new(0.6, 0.6)); +// } +// } diff --git a/crates/media-encoders/Cargo.toml b/crates/enc-avfoundation/Cargo.toml similarity index 57% rename from crates/media-encoders/Cargo.toml rename to crates/enc-avfoundation/Cargo.toml index 311f6496c7..e352610e05 100644 --- a/crates/media-encoders/Cargo.toml +++ b/crates/enc-avfoundation/Cargo.toml @@ -1,21 +1,15 @@ [package] -name = "cap-media-encoders" +name = "cap-enc-avfoundation" version = "0.1.0" edition = "2024" [dependencies] -cap-project = { path = "../project" } -cap-flags = { path = "../flags" } -cap-audio = { path = "../audio" } -cap-fail = { path = "../fail" } cap-media-info = { path = "../media-info" } +cap-ffmpeg-utils = { path = "../ffmpeg-utils" } ffmpeg.workspace = true thiserror.workspace = true tracing.workspace = true -gifski = "1.32" -imgref = "1.10" -rgb = "0.8" [target.'cfg(target_os = "macos")'.dependencies] cidre = { workspace = true } diff --git a/crates/enc-avfoundation/src/lib.rs b/crates/enc-avfoundation/src/lib.rs new file mode 100644 index 0000000000..1db0f16db5 --- /dev/null +++ b/crates/enc-avfoundation/src/lib.rs @@ -0,0 +1,5 @@ +#![cfg(target_os = "macos")] + +mod mp4; + +pub use mp4::*; diff --git a/crates/media-encoders/src/mp4_avassetwriter.rs b/crates/enc-avfoundation/src/mp4.rs similarity index 98% rename from crates/media-encoders/src/mp4_avassetwriter.rs rename to crates/enc-avfoundation/src/mp4.rs index ecf25c6766..2092c7efba 100644 --- a/crates/media-encoders/src/mp4_avassetwriter.rs +++ b/crates/enc-avfoundation/src/mp4.rs @@ -1,20 +1,20 @@ -use arc::Retained; -use cap_media_info::{AudioInfo, PlanarData, VideoInfo}; +use cap_ffmpeg_utils::PlanarData; +use cap_media_info::{AudioInfo, VideoInfo}; use cidre::{cm::SampleTimingInfo, objc::Obj, *}; use ffmpeg::{ffi::AV_TIME_BASE_Q, frame}; use std::path::PathBuf; use tracing::{debug, info}; -pub struct MP4AVAssetWriterEncoder { +pub struct MP4Encoder { #[allow(unused)] tag: &'static str, #[allow(unused)] last_pts: Option, #[allow(unused)] config: VideoInfo, - asset_writer: Retained, - video_input: Retained, - audio_input: Option>, + asset_writer: arc::R, + video_input: arc::R, + audio_input: Option>, start_time: cm::Time, first_timestamp: Option, segment_first_timestamp: Option, @@ -66,7 +66,7 @@ pub enum QueueAudioFrameError { Failed, } -impl MP4AVAssetWriterEncoder { +impl MP4Encoder { pub fn init( tag: &'static str, video_config: VideoInfo, diff --git a/crates/enc-ffmpeg/Cargo.toml b/crates/enc-ffmpeg/Cargo.toml new file mode 100644 index 0000000000..17605eb6f8 --- /dev/null +++ b/crates/enc-ffmpeg/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "cap-enc-ffmpeg" +version = "0.1.0" +edition = "2024" + +[dependencies] +cap-media-info = { path = "../media-info" } + +ffmpeg.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[lints] +workspace = true diff --git a/crates/media-encoders/src/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rs similarity index 100% rename from crates/media-encoders/src/aac.rs rename to crates/enc-ffmpeg/src/audio/aac.rs index cc0ca00da1..3df667a388 100644 --- a/crates/media-encoders/src/aac.rs +++ b/crates/enc-ffmpeg/src/audio/aac.rs @@ -1,6 +1,3 @@ -use std::collections::VecDeque; - -use crate::AudioEncoder; use cap_media_info::{AudioInfo, FFRational}; use ffmpeg::{ codec::{context, encoder}, @@ -8,6 +5,9 @@ use ffmpeg::{ frame, threading::Config, }; +use std::collections::VecDeque; + +use crate::AudioEncoder; #[derive(thiserror::Error, Debug)] pub enum AACEncoderError { diff --git a/crates/media-encoders/src/audio_encoder.rs b/crates/enc-ffmpeg/src/audio/audio_encoder.rs similarity index 100% rename from crates/media-encoders/src/audio_encoder.rs rename to crates/enc-ffmpeg/src/audio/audio_encoder.rs diff --git a/crates/enc-ffmpeg/src/audio/mod.rs b/crates/enc-ffmpeg/src/audio/mod.rs new file mode 100644 index 0000000000..529c8c28f3 --- /dev/null +++ b/crates/enc-ffmpeg/src/audio/mod.rs @@ -0,0 +1,8 @@ +mod audio_encoder; +pub use audio_encoder::*; + +mod opus; +pub use opus::*; + +mod aac; +pub use aac::*; diff --git a/crates/media-encoders/src/opus.rs b/crates/enc-ffmpeg/src/audio/opus.rs similarity index 91% rename from crates/media-encoders/src/opus.rs rename to crates/enc-ffmpeg/src/audio/opus.rs index 1c3bfcbc7c..aa6265a061 100644 --- a/crates/media-encoders/src/opus.rs +++ b/crates/enc-ffmpeg/src/audio/opus.rs @@ -5,41 +5,10 @@ use ffmpeg::{ frame, threading::Config, }; -use std::{collections::VecDeque, path::PathBuf}; +use std::collections::VecDeque; use super::AudioEncoder; -pub struct OggFile { - encoder: OpusEncoder, - output: format::context::Output, -} - -impl OggFile { - pub fn init( - mut output: PathBuf, - encoder: impl FnOnce(&mut format::context::Output) -> Result, - ) -> Result> { - output.set_extension("ogg"); - let mut output = format::output(&output)?; - - let encoder = encoder(&mut output)?; - - // make sure this happens after adding all encoders! - output.write_header()?; - - Ok(Self { encoder, output }) - } - - pub fn queue_frame(&mut self, frame: frame::Audio) { - self.encoder.queue_frame(frame, &mut self.output); - } - - pub fn finish(&mut self) { - self.encoder.finish(&mut self.output); - self.output.write_trailer().unwrap(); - } -} - pub struct OpusEncoder { #[allow(unused)] tag: &'static str, diff --git a/crates/enc-ffmpeg/src/lib.rs b/crates/enc-ffmpeg/src/lib.rs new file mode 100644 index 0000000000..d8d72d26df --- /dev/null +++ b/crates/enc-ffmpeg/src/lib.rs @@ -0,0 +1,8 @@ +mod audio; +pub use audio::*; + +mod video; +pub use video::*; + +mod mux; +pub use mux::*; diff --git a/crates/enc-ffmpeg/src/mux/mod.rs b/crates/enc-ffmpeg/src/mux/mod.rs new file mode 100644 index 0000000000..09078ff0fd --- /dev/null +++ b/crates/enc-ffmpeg/src/mux/mod.rs @@ -0,0 +1,5 @@ +mod mp4; +pub use mp4::*; + +mod ogg; +pub use ogg::*; diff --git a/crates/media-encoders/src/mp4.rs b/crates/enc-ffmpeg/src/mux/mp4.rs similarity index 91% rename from crates/media-encoders/src/mp4.rs rename to crates/enc-ffmpeg/src/mux/mp4.rs index b65ab8c1f1..577de61294 100644 --- a/crates/media-encoders/src/mp4.rs +++ b/crates/enc-ffmpeg/src/mux/mp4.rs @@ -1,14 +1,12 @@ use cap_media_info::RawVideoFormat; -use ffmpeg::{ - format::{self}, - frame, -}; +use ffmpeg::{format, frame}; use std::path::PathBuf; use tracing::{info, trace}; -use crate::H264EncoderError; - -use super::{AudioEncoder, H264Encoder}; +use crate::{ + audio::AudioEncoder, + video::{H264Encoder, H264EncoderError}, +}; pub struct MP4File { #[allow(unused)] @@ -113,6 +111,14 @@ impl MP4File { tracing::error!("Failed to write MP4 trailer: {:?}", e); } } + + pub fn video(&self) -> &H264Encoder { + &self.video + } + + pub fn video_mut(&mut self) -> &mut H264Encoder { + &mut self.video + } } pub struct MP4Input { diff --git a/crates/enc-ffmpeg/src/mux/ogg.rs b/crates/enc-ffmpeg/src/mux/ogg.rs new file mode 100644 index 0000000000..d45b6b813a --- /dev/null +++ b/crates/enc-ffmpeg/src/mux/ogg.rs @@ -0,0 +1,35 @@ +use ffmpeg::{format, frame}; +use std::path::PathBuf; + +use crate::audio::{OpusEncoder, OpusEncoderError}; + +pub struct OggFile { + encoder: OpusEncoder, + output: format::context::Output, +} + +impl OggFile { + pub fn init( + mut output: PathBuf, + encoder: impl FnOnce(&mut format::context::Output) -> Result, + ) -> Result> { + output.set_extension("ogg"); + let mut output = format::output(&output)?; + + let encoder = encoder(&mut output)?; + + // make sure this happens after adding all encoders! + output.write_header()?; + + Ok(Self { encoder, output }) + } + + pub fn queue_frame(&mut self, frame: frame::Audio) { + self.encoder.queue_frame(frame, &mut self.output); + } + + pub fn finish(&mut self) { + self.encoder.finish(&mut self.output); + self.output.write_trailer().unwrap(); + } +} diff --git a/crates/media-encoders/src/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs similarity index 100% rename from crates/media-encoders/src/h264.rs rename to crates/enc-ffmpeg/src/video/h264.rs diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs new file mode 100644 index 0000000000..f793b8cbb0 --- /dev/null +++ b/crates/enc-ffmpeg/src/video/mod.rs @@ -0,0 +1,2 @@ +mod h264; +pub use h264::*; diff --git a/crates/enc-gif/Cargo.toml b/crates/enc-gif/Cargo.toml new file mode 100644 index 0000000000..cea52b600e --- /dev/null +++ b/crates/enc-gif/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "cap-enc-gif" +version = "0.1.0" +edition = "2024" + +[dependencies] +thiserror.workspace = true +tracing.workspace = true +gifski = "1.32" +imgref = "1.10" +rgb = "0.8" + +[lints] +workspace = true diff --git a/crates/media-encoders/src/gif.rs b/crates/enc-gif/src/lib.rs similarity index 100% rename from crates/media-encoders/src/gif.rs rename to crates/enc-gif/src/lib.rs diff --git a/crates/enc-mediafoundation/Cargo.toml b/crates/enc-mediafoundation/Cargo.toml new file mode 100644 index 0000000000..1cc679fdb5 --- /dev/null +++ b/crates/enc-mediafoundation/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "cap-enc-mediafoundation" +version = "0.1.0" +edition = "2024" + +[dependencies] +cap-media-info = { path = "../media-info" } +futures.workspace = true +thiserror.workspace = true + +[target.'cfg(windows)'.dependencies] +windows = { workspace = true, features = [ + "System", + "Graphics_Capture", + "Graphics_DirectX_Direct3D11", + "Foundation_Metadata", + "Win32_Graphics_Gdi", + "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi_Common", + "Win32_Graphics_Dxgi", + "Win32_System_Com", + "Win32_System_Com_StructuredStorage", + "Win32_System_Ole", + "Win32_System_Threading", + "Win32_System_WinRT_Direct3D11", + "Win32_System_WinRT_Graphics_Capture", + "Win32_UI_WindowsAndMessaging", + "Win32_Media_MediaFoundation", + "Win32_Media_DxMediaObjects", + "Win32_Foundation", + "Foundation_Collections", + "Win32_System_Variant", + "Storage_Search", + "Storage_Streams", + "Win32_UI_Input", + "Media_Core", + "Media_MediaProperties", + "Media_Transcoding", + "Win32_Storage_FileSystem", + "Win32_System_Diagnostics_Debug", + "Win32_UI_Input_KeyboardAndMouse", + "Security_Authorization_AppCapabilityAccess", +] } +windows-core.workspace = true +windows-numerics = "0.2.0" +scap-direct3d = { path = "../scap-direct3d" } + +[dev-dependencies] +scap-targets = { path = "../scap-targets" } +clap = { version = "4.5.46", features = ["derive"] } + +[lints] +workspace = true diff --git a/crates/enc-mediafoundation/examples/cli.rs b/crates/enc-mediafoundation/examples/cli.rs new file mode 100644 index 0000000000..6e5f97f1c6 --- /dev/null +++ b/crates/enc-mediafoundation/examples/cli.rs @@ -0,0 +1,397 @@ +fn main() { + #[cfg(windows)] + win::main(); +} + +#[cfg(windows)] +mod win { + use args::Args; + use cap_enc_mediafoundation::{ + d3d::create_d3d_device, + media::MF_VERSION, + video::{H264Encoder, InputSample, SampleWriter}, + }; + use clap::Parser; + use scap_targets::Display; + use std::{path::Path, sync::Arc, time::Duration}; + use windows::{ + Foundation::{Metadata::ApiInformation, TimeSpan}, + Graphics::Capture::GraphicsCaptureSession, + Win32::{ + Media::MediaFoundation::{self, MFSTARTUP_FULL, MFStartup}, + System::{ + Diagnostics::Debug::{DebugBreak, IsDebuggerPresent}, + Threading::GetCurrentProcessId, + WinRT::{RO_INIT_MULTITHREADED, RoInitialize}, + }, + }, + core::{HSTRING, Result, RuntimeName}, + }; + + #[allow(clippy::too_many_arguments)] + fn run( + display_index: usize, + output_path: &str, + bit_rate: u32, + frame_rate: u32, + resolution: Resolution, + verbose: bool, + wait_for_debugger: bool, + ) -> Result<()> { + unsafe { + RoInitialize(RO_INIT_MULTITHREADED)?; + } + unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL)? } + + if wait_for_debugger { + let pid = unsafe { GetCurrentProcessId() }; + println!("Waiting for a debugger to attach (PID: {})...", pid); + loop { + if unsafe { IsDebuggerPresent().into() } { + break; + } + std::thread::sleep(Duration::from_secs(1)); + } + unsafe { + DebugBreak(); + } + } + + // Check to make sure Windows.Graphics.Capture is available + if !required_capture_features_supported()? { + exit_with_error( + "The required screen capture features are not supported on this device for this release of Windows!\nPlease update your operating system (minimum: Windows 10 Version 1903, Build 18362).", + ); + } + + if verbose { + println!( + "Using index \"{}\" and path \"{}\".", + display_index, output_path + ); + } + + let item = Display::primary() + .raw_handle() + .try_as_capture_item() + .unwrap(); + + // Resolve encoding settings + let resolution = if let Some(resolution) = resolution.get_size() { + resolution + } else { + item.Size()? + }; + let bit_rate = bit_rate * 1000000; + + // Start the recording + { + let d3d_device = create_d3d_device()?; + + let (frame_tx, frame_rx) = std::sync::mpsc::channel(); + + let mut first_time = None; + let mut capturer = scap_direct3d::Capturer::new( + item, + scap_direct3d::Settings { + is_border_required: Some(false), + ..Default::default() + }, + { + let frame_tx = frame_tx.clone(); + move |frame| { + let frame_time = frame.inner().SystemRelativeTime()?; + + let first_time = first_time.get_or_insert(frame_time); + let timestamp = TimeSpan { + Duration: frame_time.Duration - first_time.Duration, + }; + + let _ = frame_tx.send(Some(frame)); + + Ok(()) + } + }, + move || { + let _ = frame_tx.send(None); + + Ok(()) + }, + Some(d3d_device.clone()), + ) + .unwrap(); + + let mut video_encoder = H264Encoder::new( + &d3d_device, + capturer.settings().pixel_format.as_dxgi(), + resolution, + frame_rate, + 0.07, + ) + .unwrap(); + + let output_path = std::env::current_dir().unwrap().join(output_path); + + let sample_writer = Arc::new(SampleWriter::new(output_path.as_path())?); + + let stream_index = sample_writer.add_stream(&video_encoder.output_type())?; + + capturer.start()?; + sample_writer.start()?; + + std::thread::spawn({ + let sample_writer = sample_writer.clone(); + move || { + unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) }.unwrap(); + + video_encoder.start().unwrap(); + + while let Ok(e) = video_encoder.get_event() { + match e { + MediaFoundation::METransformNeedInput => { + let Some(frame) = frame_rx.recv().unwrap() else { + break; + }; + + if video_encoder + .handle_needs_input( + frame.texture(), + frame.inner().SystemRelativeTime().unwrap(), + ) + .is_err() + { + break; + } + } + MediaFoundation::METransformHaveOutput => { + if let Some(output_sample) = + video_encoder.handle_has_output().unwrap() + { + sample_writer.write(stream_index, &output_sample).unwrap(); + } + } + _ => {} + } + } + + video_encoder.finish().unwrap(); + } + }); + + pause(); + + capturer.stop().unwrap(); + sample_writer.stop()?; + } + + Ok(()) + } + + pub fn main() { + // Handle /? + let args: Vec<_> = std::env::args().collect(); + if args.contains(&"/?".to_owned()) || args.contains(&"-?".to_owned()) { + Args::parse_from(["displayrecorder.exe", "--help"]); + std::process::exit(0); + } + + let args = Args::parse(); + + if let Some(command) = args.command { + match command { + args::Commands::EnumEncoders => enum_encoders().unwrap(), + } + return; + } + + let monitor_index: usize = args.display; + let output_path = args.output_file.as_str(); + let verbose = args.verbose; + let wait_for_debugger = args.wait_for_debugger; + let bit_rate: u32 = args.bit_rate; + let frame_rate: u32 = args.frame_rate; + let resolution: Resolution = args.resolution; + + // Validate some of the params + if !validate_path(output_path) { + exit_with_error("Invalid path specified!"); + } + + let result = run( + monitor_index, + output_path, + bit_rate, + frame_rate, + resolution, + verbose | wait_for_debugger, + wait_for_debugger, + ); + + // We do this for nicer HRESULT printing when errors occur. + if let Err(error) = result { + error.code().unwrap(); + } + } + + fn pause() { + println!("Press ENTER to stop recording..."); + std::io::Read::read(&mut std::io::stdin(), &mut [0]).unwrap(); + } + + fn enum_encoders() -> Result<()> { + let encoder_devices = VideoEncoderDevice::enumerate()?; + if encoder_devices.is_empty() { + exit_with_error("No hardware H264 encoders found!"); + } + println!("Encoders ({}):", encoder_devices.len()); + for (i, encoder_device) in encoder_devices.iter().enumerate() { + println!(" {} - {}", i, encoder_device.display_name()); + } + Ok(()) + } + + fn validate_path>(path: P) -> bool { + let path = path.as_ref(); + let mut valid = true; + if let Some(extension) = path.extension() { + if extension != "mp4" { + valid = false; + } + } else { + valid = false; + } + valid + } + + fn exit_with_error(message: &str) -> ! { + println!("{}", message); + std::process::exit(1); + } + + fn win32_programmatic_capture_supported() -> Result { + ApiInformation::IsApiContractPresentByMajor( + &HSTRING::from("Windows.Foundation.UniversalApiContract"), + 8, + ) + } + + fn required_capture_features_supported() -> Result { + let result = ApiInformation::IsTypePresent(&HSTRING::from(GraphicsCaptureSession::NAME))? && // Windows.Graphics.Capture is present + GraphicsCaptureSession::IsSupported()? && // The CaptureService is available + win32_programmatic_capture_supported()?; + Ok(result) + } + + mod args { + use clap::{Parser, Subcommand}; + + use cap_enc_mediafoundation::resolution::Resolution; + + #[derive(Parser, Debug)] + #[clap(author, version, about, long_about = None)] + pub struct Args { + /// The index of the display you'd like to record. + #[clap(short, long, default_value_t = 0)] + pub display: usize, + + /// The bit rate you would like to encode at (in Mbps). + #[clap(short, long, default_value_t = 18)] + pub bit_rate: u32, + + /// The frame rate you would like to encode at. + #[clap(short, long, default_value_t = 60)] + pub frame_rate: u32, + + /// The resolution you would like to encode at: native, 720p, 1080p, 2160p, or 4320p. + #[clap(short, long, default_value_t = Resolution::Native)] + pub resolution: Resolution, + + /// The index of the encoder you'd like to use to record (use enum-encoders command for a list of encoders and their indices). + #[clap(short, long, default_value_t = 0)] + pub encoder: usize, + + /// Disables the yellow capture border (only available on Windows 11). + #[clap(long)] + pub borderless: bool, + + /// Enables verbose (debug) output. + #[clap(short, long)] + pub verbose: bool, + + /// The program will wait for a debugger to attach before starting. + #[clap(long)] + pub wait_for_debugger: bool, + + /// Recording immediately starts. End the recording through console input. + #[clap(long)] + pub console_mode: bool, + + /// The output file that will contain the recording. + #[clap(default_value = "recording.mp4")] + pub output_file: String, + + /// Subcommands to execute. + #[clap(subcommand)] + pub command: Option, + } + + #[derive(Subcommand, Debug)] + #[clap(args_conflicts_with_subcommands = true)] + pub enum Commands { + /// Lists the available hardware H264 encoders. + EnumEncoders, + } + } + + mod hotkey { + use std::sync::atomic::{AtomicI32, Ordering}; + use windows::{ + Win32::UI::Input::KeyboardAndMouse::{ + HOT_KEY_MODIFIERS, RegisterHotKey, UnregisterHotKey, + }, + core::Result, + }; + + static HOT_KEY_ID: AtomicI32 = AtomicI32::new(0); + + pub struct HotKey { + id: i32, + } + + impl HotKey { + pub fn new(modifiers: HOT_KEY_MODIFIERS, key: u32) -> Result { + let id = HOT_KEY_ID.fetch_add(1, Ordering::SeqCst) + 1; + unsafe { + RegisterHotKey(None, id, modifiers, key)?; + } + Ok(Self { id }) + } + } + + impl Drop for HotKey { + fn drop(&mut self) { + unsafe { UnregisterHotKey(None, self.id).ok().unwrap() } + } + } + } + + #[cfg(test)] + mod tests { + use crate::validate_path; + + #[test] + fn path_parsing_test() { + assert!(validate_path("something.mp4")); + assert!(validate_path("somedir/something.mp4")); + assert!(validate_path("somedir\\something.mp4")); + assert!(validate_path("../something.mp4")); + + assert!(!validate_path(".")); + assert!(!validate_path("*")); + assert!(!validate_path("something")); + assert!(!validate_path(".mp4")); + assert!(!validate_path("mp4")); + assert!(!validate_path("something.avi")); + } + } +} diff --git a/crates/enc-mediafoundation/src/d3d.rs b/crates/enc-mediafoundation/src/d3d.rs new file mode 100644 index 0000000000..81bf3917ee --- /dev/null +++ b/crates/enc-mediafoundation/src/d3d.rs @@ -0,0 +1,74 @@ +use windows::Graphics::DirectX::Direct3D11::{IDirect3DDevice, IDirect3DSurface}; +use windows::Win32::Foundation::HMODULE; +use windows::Win32::Graphics::Direct3D11::{D3D11_CREATE_DEVICE_DEBUG, ID3D11Texture2D}; +use windows::Win32::Graphics::Dxgi::IDXGISurface; +use windows::Win32::Graphics::{ + Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}, + Direct3D11::{ + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG, D3D11_SDK_VERSION, + D3D11CreateDevice, ID3D11Device, + }, + Dxgi::{DXGI_ERROR_UNSUPPORTED, IDXGIDevice}, +}; +use windows::Win32::System::WinRT::Direct3D11::{ + CreateDirect3D11DeviceFromDXGIDevice, CreateDirect3D11SurfaceFromDXGISurface, + IDirect3DDxgiInterfaceAccess, +}; +use windows::core::{Interface, Result}; + +fn create_d3d_device_with_type( + driver_type: D3D_DRIVER_TYPE, + flags: D3D11_CREATE_DEVICE_FLAG, + device: *mut Option, +) -> Result<()> { + unsafe { + D3D11CreateDevice( + None, + driver_type, + HMODULE(std::ptr::null_mut()), + flags, + None, + D3D11_SDK_VERSION, + Some(device), + None, + None, + ) + } +} + +pub fn create_d3d_device() -> Result { + let mut device = None; + let flags = { + let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + if cfg!(feature = "d3ddebug") { + flags |= D3D11_CREATE_DEVICE_DEBUG; + } + flags + }; + let mut result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, flags, &mut device); + if let Err(error) = &result { + if error.code() == DXGI_ERROR_UNSUPPORTED { + result = create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device); + } + } + result?; + Ok(device.unwrap()) +} + +pub fn create_direct3d_device(d3d_device: &ID3D11Device) -> Result { + let dxgi_device: IDXGIDevice = d3d_device.cast()?; + let inspectable = unsafe { CreateDirect3D11DeviceFromDXGIDevice(Some(&dxgi_device))? }; + inspectable.cast() +} + +pub fn create_direct3d_surface(d3d_texture: &ID3D11Texture2D) -> Result { + let dxgi_surface: IDXGISurface = d3d_texture.cast()?; + let inspectable = unsafe { CreateDirect3D11SurfaceFromDXGISurface(Some(&dxgi_surface))? }; + inspectable.cast() +} + +pub fn get_d3d_interface_from_object(object: &S) -> Result { + let access: IDirect3DDxgiInterfaceAccess = object.cast()?; + let object = unsafe { access.GetInterface::()? }; + Ok(object) +} diff --git a/crates/enc-mediafoundation/src/lib.rs b/crates/enc-mediafoundation/src/lib.rs new file mode 100644 index 0000000000..f1d1eb7571 --- /dev/null +++ b/crates/enc-mediafoundation/src/lib.rs @@ -0,0 +1,9 @@ +#![cfg(windows)] + +pub mod d3d; +pub mod media; +mod mft; +mod unsafe_send; +pub mod video; + +pub use video::H264Encoder; diff --git a/crates/enc-mediafoundation/src/media.rs b/crates/enc-mediafoundation/src/media.rs new file mode 100644 index 0000000000..d7f57a9f22 --- /dev/null +++ b/crates/enc-mediafoundation/src/media.rs @@ -0,0 +1,43 @@ +use windows::{ + Win32::Media::MediaFoundation::IMFAttributes, + core::{GUID, Result}, +}; + +// These inlined helpers aren't represented in the metadata + +// This is the value for Win7+ +pub const MF_VERSION: u32 = 131184; + +fn pack_2_u32_as_u64(high: u32, low: u32) -> u64 { + ((high as u64) << 32) | low as u64 +} + +#[allow(non_snake_case)] +unsafe fn MFSetAttribute2UINT32asUINT64( + attributes: &IMFAttributes, + key: &GUID, + high: u32, + low: u32, +) -> Result<()> { + unsafe { attributes.SetUINT64(key, pack_2_u32_as_u64(high, low)) } +} + +#[allow(non_snake_case)] +pub unsafe fn MFSetAttributeSize( + attributes: &IMFAttributes, + key: &GUID, + width: u32, + height: u32, +) -> Result<()> { + unsafe { MFSetAttribute2UINT32asUINT64(attributes, key, width, height) } +} + +#[allow(non_snake_case)] +pub unsafe fn MFSetAttributeRatio( + attributes: &IMFAttributes, + key: &GUID, + numerator: u32, + denominator: u32, +) -> Result<()> { + unsafe { MFSetAttribute2UINT32asUINT64(attributes, key, numerator, denominator) } +} diff --git a/crates/enc-mediafoundation/src/mft.rs b/crates/enc-mediafoundation/src/mft.rs new file mode 100644 index 0000000000..5a0babf7ba --- /dev/null +++ b/crates/enc-mediafoundation/src/mft.rs @@ -0,0 +1,112 @@ +use windows::{ + Win32::Media::MediaFoundation::{ + IMFActivate, IMFAttributes, MF_E_ATTRIBUTENOTFOUND, MFT_ENUM_FLAG, MFT_REGISTER_TYPE_INFO, + MFTEnumEx, + }, + core::{Array, GUID, Result}, +}; +use windows::{ + Win32::Media::MediaFoundation::{ + IMFTransform, MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE, + MFT_ENUM_FLAG_SORTANDFILTER, MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_FRIENDLY_NAME_Attribute, + }, + core::Interface, +}; + +#[derive(Clone)] +pub struct EncoderDevice { + source: IMFActivate, + display_name: String, +} + +impl EncoderDevice { + pub fn enumerate(major_type: GUID, subtype: GUID) -> Result> { + let output_info = MFT_REGISTER_TYPE_INFO { + guidMajorType: major_type, + guidSubtype: subtype, + }; + let encoders = enumerate_mfts( + &MFT_CATEGORY_VIDEO_ENCODER, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY | MFT_ENUM_FLAG_SORTANDFILTER, + None, + Some(&output_info), + )?; + let mut encoder_devices = Vec::new(); + for encoder in encoders { + let display_name = if let Some(display_name) = + get_string_attribute(&encoder.cast()?, &MFT_FRIENDLY_NAME_Attribute)? + { + display_name + } else { + "Unknown".to_owned() + }; + let encoder_device = EncoderDevice { + source: encoder, + display_name, + }; + encoder_devices.push(encoder_device); + } + Ok(encoder_devices) + } + + pub fn display_name(&self) -> &str { + &self.display_name + } + + pub fn create_transform(&self) -> Result { + unsafe { self.source.ActivateObject() } + } +} + +fn enumerate_mfts( + category: &GUID, + flags: MFT_ENUM_FLAG, + input_type: Option<&MFT_REGISTER_TYPE_INFO>, + output_type: Option<&MFT_REGISTER_TYPE_INFO>, +) -> Result> { + let mut transform_sources = Vec::new(); + let mfactivate_list = unsafe { + let mut data = std::ptr::null_mut(); + let mut len = 0; + MFTEnumEx( + *category, + flags, + input_type.map(|info| info as *const _), + output_type.map(|info| info as *const _), + &mut data, + &mut len, + )?; + Array::::from_raw_parts(data as _, len) + }; + if !mfactivate_list.is_empty() { + for mfactivate in mfactivate_list.as_slice() { + if let Some(transform_source) = mfactivate.clone() { + transform_sources.push(transform_source); + } + } + } + Ok(transform_sources) +} + +fn get_string_attribute( + attributes: &IMFAttributes, + attribute_guid: &GUID, +) -> Result> { + unsafe { + match attributes.GetStringLength(attribute_guid) { + Ok(mut length) => { + let mut result = vec![0u16; (length + 1) as usize]; + attributes.GetString(attribute_guid, &mut result, Some(&mut length))?; + result.resize(length as usize, 0); + Ok(String::from_utf16(&result).ok()) + } + Err(error) => { + if error.code() == MF_E_ATTRIBUTENOTFOUND { + Ok(None) + } else { + Err(error) + } + } + } + } +} diff --git a/crates/enc-mediafoundation/src/unsafe_send.rs b/crates/enc-mediafoundation/src/unsafe_send.rs new file mode 100644 index 0000000000..a4e831982a --- /dev/null +++ b/crates/enc-mediafoundation/src/unsafe_send.rs @@ -0,0 +1,39 @@ +use std::{fmt::Display, ops::Deref}; + +#[derive(Debug)] +pub struct UnsafeSend(pub T); + +unsafe impl Send for UnsafeSend {} +unsafe impl Sync for UnsafeSend {} + +impl Deref for UnsafeSend { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From for UnsafeSend { + fn from(inner: T) -> Self { + Self(inner) + } +} + +impl Clone for UnsafeSend +where + T: Clone, +{ + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl Display for UnsafeSend +where + T: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs new file mode 100644 index 0000000000..f7b7222afa --- /dev/null +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -0,0 +1,395 @@ +use windows::{ + Foundation::TimeSpan, + Graphics::SizeInt32, + Win32::{ + Foundation::E_NOTIMPL, + Graphics::{ + Direct3D11::{ID3D11Device, ID3D11Texture2D}, + Dxgi::Common::{DXGI_FORMAT, DXGI_FORMAT_NV12}, + }, + Media::MediaFoundation::{ + IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, IMFSample, + IMFTransform, MEDIA_EVENT_GENERATOR_GET_EVENT_FLAGS, MF_E_INVALIDMEDIATYPE, + MF_E_NO_MORE_TYPES, MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_TYPE, + MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, + MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, + MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, + MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType, + MFCreateSample, MFMediaType_Video, MFT_MESSAGE_COMMAND_FLUSH, + MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_OF_STREAM, + MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, + MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY, + MFVideoFormat_H264, MFVideoFormat_NV12, MFVideoInterlace_Progressive, + }, + }, + core::{Error, Interface}, +}; + +use crate::{ + media::{MFSetAttributeRatio, MFSetAttributeSize}, + mft::EncoderDevice, + video::{NewVideoProcessorError, VideoProcessor}, +}; + +pub struct VideoEncoderOutputSample { + sample: IMFSample, +} + +impl VideoEncoderOutputSample { + pub fn sample(&self) -> &IMFSample { + &self.sample + } +} + +pub struct H264Encoder { + _d3d_device: ID3D11Device, + _media_device_manager: IMFDXGIDeviceManager, + _device_manager_reset_token: u32, + + video_processor: VideoProcessor, + + transform: IMFTransform, + event_generator: IMFMediaEventGenerator, + input_stream_id: u32, + output_stream_id: u32, + output_type: IMFMediaType, + bitrate: u32, + + first_time: Option, +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum NewVideoEncoderError { + #[error("NoVideoEncoderDevice")] + NoVideoEncoderDevice, + #[error("EncoderTransform: {0}")] + EncoderTransform(windows::core::Error), + #[error("VideoProcessor: {0}")] + VideoProcessor(NewVideoProcessorError), + #[error("DeviceManager: {0}")] + DeviceManager(windows::core::Error), + #[error("EventGenerator: {0}")] + EventGenerator(windows::core::Error), + #[error("ConfigureStreams: {0}")] + ConfigureStreams(windows::core::Error), + #[error("OutputType: {0}")] + OutputType(windows::core::Error), + #[error("InputType: {0}")] + InputType(windows::core::Error), +} + +unsafe impl Send for H264Encoder {} + +impl H264Encoder { + pub fn new_with_scaled_output( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + let bitrate = calculate_bitrate( + output_resolution.Width as u32, + output_resolution.Height as u32, + frame_rate, + bitrate_multipler, + ); + + let transform = EncoderDevice::enumerate(MFMediaType_Video, MFVideoFormat_H264) + .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)? + .first() + .cloned() + .ok_or(NewVideoEncoderError::NoVideoEncoderDevice)? + .create_transform() + .map_err(NewVideoEncoderError::EncoderTransform)?; + + let video_processor = VideoProcessor::new( + d3d_device.clone(), + format, + input_resolution, + DXGI_FORMAT_NV12, + output_resolution, + frame_rate, + ) + .map_err(NewVideoEncoderError::VideoProcessor)?; + + // Create MF device manager + let mut device_manager_reset_token: u32 = 0; + let media_device_manager = { + let mut media_device_manager = None; + unsafe { + MFCreateDXGIDeviceManager( + &mut device_manager_reset_token, + &mut media_device_manager, + ) + .map_err(NewVideoEncoderError::DeviceManager)? + }; + media_device_manager.expect("Device manager unexpectedly None") + }; + unsafe { + media_device_manager + .ResetDevice(d3d_device, device_manager_reset_token) + .map_err(NewVideoEncoderError::DeviceManager)? + }; + + // Setup MFTransform + let event_generator: IMFMediaEventGenerator = transform + .cast() + .map_err(NewVideoEncoderError::EventGenerator)?; + let attributes = unsafe { + transform + .GetAttributes() + .map_err(NewVideoEncoderError::EventGenerator)? + }; + unsafe { + attributes + .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + .map_err(NewVideoEncoderError::EventGenerator)?; + attributes + .SetUINT32(&MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1) + .map_err(NewVideoEncoderError::EventGenerator)?; + }; + + let mut number_of_input_streams = 0; + let mut number_of_output_streams = 0; + unsafe { + transform + .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams) + .map_err(NewVideoEncoderError::EventGenerator)? + }; + let (input_stream_ids, output_stream_ids) = { + let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; + let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; + let result = + unsafe { transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) }; + match result { + Ok(_) => {} + Err(error) => { + // https://docs.microsoft.com/en-us/windows/win32/api/mftransform/nf-mftransform-imftransform-getstreamids + // This method can return E_NOTIMPL if both of the following conditions are true: + // * The transform has a fixed number of streams. + // * The streams are numbered consecutively from 0 to n – 1, where n is the + // number of input streams or output streams. In other words, the first + // input stream is 0, the second is 1, and so on; and the first output + // stream is 0, the second is 1, and so on. + if error.code() == E_NOTIMPL { + for i in 0..number_of_input_streams { + input_stream_ids[i as usize] = i; + } + for i in 0..number_of_output_streams { + output_stream_ids[i as usize] = i; + } + } else { + return Err(NewVideoEncoderError::ConfigureStreams(error)); + } + } + } + (input_stream_ids, output_stream_ids) + }; + let input_stream_id = input_stream_ids[0]; + let output_stream_id = output_stream_ids[0]; + + // TOOD: Avoid this AddRef? + unsafe { + let temp = media_device_manager.clone(); + transform + .ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, std::mem::transmute(temp)) + .map_err(NewVideoEncoderError::EncoderTransform)?; + }; + + let output_type = (|| unsafe { + let output_type = MFCreateMediaType()?; + let attributes: IMFAttributes = output_type.cast()?; + output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; + output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; + output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; + transform.SetOutputType(output_stream_id, &output_type, 0)?; + Ok(output_type) + })() + .map_err(NewVideoEncoderError::OutputType)?; + + let input_type: Option = (|| unsafe { + let mut count = 0; + loop { + let result = transform.GetInputAvailableType(input_stream_id, count); + if let Err(error) = &result { + if error.code() == MF_E_NO_MORE_TYPES { + break Ok(None); + } + } + + let input_type = result?; + let attributes: IMFAttributes = input_type.cast()?; + input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + let result = transform.SetInputType( + input_stream_id, + &input_type, + MFT_SET_TYPE_TEST_ONLY.0 as u32, + ); + if let Err(error) = &result { + if error.code() == MF_E_INVALIDMEDIATYPE { + count += 1; + continue; + } + } + result?; + break Ok(Some(input_type)); + } + })() + .map_err(NewVideoEncoderError::InputType)?; + if let Some(input_type) = input_type { + unsafe { transform.SetInputType(input_stream_id, &input_type, 0) } + .map_err(NewVideoEncoderError::InputType)?; + } else { + return Err(NewVideoEncoderError::InputType(Error::new( + MF_E_TRANSFORM_TYPE_NOT_SET, + "No suitable input type found! Try a different set of encoding settings.", + ))); + } + + Ok(Self { + _d3d_device: d3d_device.clone(), + _media_device_manager: media_device_manager, + _device_manager_reset_token: device_manager_reset_token, + + video_processor, + + transform, + event_generator, + input_stream_id, + output_stream_id, + bitrate, + + output_type, + first_time: None, + }) + } + + pub fn new( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output( + d3d_device, + format, + resolution, + resolution, + frame_rate, + bitrate_multipler, + ) + } + + pub fn bitrate(&self) -> u32 { + self.bitrate + } + + pub fn output_type(&self) -> &IMFMediaType { + &self.output_type + } + + pub fn finish(&self) -> windows::core::Result<()> { + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + } + Ok(()) + } + + pub fn start(&self) -> windows::core::Result<()> { + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)?; + } + + Ok(()) + } + + pub fn get_event(&self) -> windows::core::Result { + let event = unsafe { + self.event_generator + .GetEvent(MEDIA_EVENT_GENERATOR_GET_EVENT_FLAGS(0))? + }; + + Ok(MF_EVENT_TYPE(unsafe { event.GetType()? } as i32)) + } + + pub fn handle_needs_input( + &mut self, + texture: &ID3D11Texture2D, + timestamp: TimeSpan, + ) -> windows::core::Result<()> { + self.video_processor.process_texture(texture)?; + + let first_time = self.first_time.get_or_insert(timestamp); + + let input_buffer = unsafe { + MFCreateDXGISurfaceBuffer( + &ID3D11Texture2D::IID, + self.video_processor.output_texture(), + 0, + false, + )? + }; + let mf_sample = unsafe { MFCreateSample()? }; + unsafe { + mf_sample.AddBuffer(&input_buffer)?; + mf_sample.SetSampleTime(timestamp.Duration - first_time.Duration)?; + self.transform + .ProcessInput(self.input_stream_id, &mf_sample, 0)?; + }; + Ok(()) + } + + pub fn handle_has_output(&mut self) -> windows::core::Result> { + let mut status = 0; + let output_buffer = MFT_OUTPUT_DATA_BUFFER { + dwStreamID: self.output_stream_id, + ..Default::default() + }; + + let sample = unsafe { + let mut output_buffers = [output_buffer]; + self.transform + .ProcessOutput(0, &mut output_buffers, &mut status)?; + output_buffers[0].pSample.as_ref().cloned() + }; + + Ok(sample) + } +} + +fn calculate_bitrate(width: u32, height: u32, fps: u32, multiplier: f32) -> u32 { + ((width * height * ((fps - 30) / 2 + 30)) as f32 * multiplier) as u32 +} diff --git a/crates/enc-mediafoundation/src/video/mod.rs b/crates/enc-mediafoundation/src/video/mod.rs new file mode 100644 index 0000000000..273711713b --- /dev/null +++ b/crates/enc-mediafoundation/src/video/mod.rs @@ -0,0 +1,5 @@ +mod h264; +mod video_processor; + +pub use h264::*; +pub use video_processor::*; diff --git a/crates/enc-mediafoundation/src/video/video_processor.rs b/crates/enc-mediafoundation/src/video/video_processor.rs new file mode 100644 index 0000000000..fe26d73f33 --- /dev/null +++ b/crates/enc-mediafoundation/src/video/video_processor.rs @@ -0,0 +1,292 @@ +use std::mem::ManuallyDrop; + +use windows::{ + Graphics::{RectInt32, SizeInt32}, + Win32::{ + Foundation::RECT, + Graphics::{ + Direct3D11::{ + D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BIND_VIDEO_ENCODER, + D3D11_TEX2D_VPIV, D3D11_TEX2D_VPOV, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, + D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, D3D11_VIDEO_PROCESSOR_COLOR_SPACE, + D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0, D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255, + D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0, D3D11_VIDEO_PROCESSOR_STREAM, + D3D11_VIDEO_USAGE_OPTIMAL_QUALITY, D3D11_VPIV_DIMENSION_TEXTURE2D, + D3D11_VPOV_DIMENSION_TEXTURE2D, ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, + ID3D11VideoContext, ID3D11VideoDevice, ID3D11VideoProcessor, + ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, + }, + Dxgi::Common::{DXGI_FORMAT, DXGI_RATIONAL, DXGI_SAMPLE_DESC}, + }, + }, + core::Interface, +}; +use windows_numerics::Vector2; + +#[derive(Clone)] +pub struct VideoProcessor { + _d3d_device: ID3D11Device, + d3d_context: ID3D11DeviceContext, + + _video_device: ID3D11VideoDevice, + video_context: ID3D11VideoContext, + video_processor: ID3D11VideoProcessor, + video_output_texture: ID3D11Texture2D, + video_output: ID3D11VideoProcessorOutputView, + video_input_texture: ID3D11Texture2D, + video_input: ID3D11VideoProcessorInputView, +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum NewVideoProcessorError { + #[error("GetDevice: {0}")] + GetDevice(windows::core::Error), + #[error("GetContext: {0}")] + GetContext(windows::core::Error), + #[error("CreateVideoProcessor: {0}")] + CreateVideoProcessor(windows::core::Error), + #[error("CreateInput: {0}")] + CreateInput(windows::core::Error), + #[error("CreateOutput: {0}")] + CreateOutput(windows::core::Error), +} + +impl VideoProcessor { + pub fn new( + d3d_device: ID3D11Device, + input_format: DXGI_FORMAT, + input_size: SizeInt32, + output_format: DXGI_FORMAT, + output_size: SizeInt32, + frame_rate: u32, + ) -> Result { + let d3d_context = unsafe { d3d_device.GetImmediateContext() } + .map_err(NewVideoProcessorError::GetDevice)?; + + // Setup video conversion + let video_device: ID3D11VideoDevice = d3d_device + .cast() + .map_err(NewVideoProcessorError::GetDevice)?; + let video_context: ID3D11VideoContext = d3d_context + .cast() + .map_err(NewVideoProcessorError::GetDevice)?; + + let video_desc = D3D11_VIDEO_PROCESSOR_CONTENT_DESC { + InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + InputFrameRate: DXGI_RATIONAL { + Numerator: frame_rate, + Denominator: 1, + }, + InputWidth: input_size.Width as u32, + InputHeight: input_size.Height as u32, + OutputFrameRate: DXGI_RATIONAL { + Numerator: frame_rate, + Denominator: 1, + }, + OutputWidth: output_size.Width as u32, + OutputHeight: output_size.Height as u32, + Usage: D3D11_VIDEO_USAGE_OPTIMAL_QUALITY, + }; + let video_enum = unsafe { video_device.CreateVideoProcessorEnumerator(&video_desc) } + .map_err(NewVideoProcessorError::CreateVideoProcessor)?; + + let video_processor = unsafe { video_device.CreateVideoProcessor(&video_enum, 0) } + .map_err(NewVideoProcessorError::CreateVideoProcessor)?; + + let mut color_space = D3D11_VIDEO_PROCESSOR_COLOR_SPACE { + _bitfield: 1 | D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255.0 as u32, + }; + unsafe { video_context.VideoProcessorSetOutputColorSpace(&video_processor, &color_space) }; + color_space._bitfield = 1 | D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235.0 as u32; + unsafe { + video_context.VideoProcessorSetStreamColorSpace(&video_processor, 0, &color_space) + }; + + // If the input and output resolutions don't match, setup the + // video processor to preserve the aspect ratio when scaling. + if input_size.Width != output_size.Width || input_size.Height != output_size.Height { + let dest_rect = compute_dest_rect(&output_size, &input_size); + let rect = RECT { + left: dest_rect.X, + top: dest_rect.Y, + right: dest_rect.X + dest_rect.Width, + bottom: dest_rect.Y + dest_rect.Height, + }; + unsafe { + video_context.VideoProcessorSetStreamDestRect( + &video_processor, + 0, + true, + Some(&rect), + ) + }; + } + + let mut texture_desc = D3D11_TEXTURE2D_DESC { + Width: output_size.Width as u32, + Height: output_size.Height as u32, + ArraySize: 1, + MipLevels: 1, + Format: output_format, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + ..Default::default() + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_VIDEO_ENCODER.0) as u32, + ..Default::default() + }; + let video_output_texture = unsafe { + let mut texture = None; + d3d_device + .CreateTexture2D(&texture_desc, None, Some(&mut texture)) + .map_err(NewVideoProcessorError::CreateOutput)?; + texture.unwrap() + }; + + let output_view_desc = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC { + ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPOV { MipSlice: 0 }, + }, + }; + let video_output = unsafe { + let mut output = None; + video_device + .CreateVideoProcessorOutputView( + &video_output_texture, + &video_enum, + &output_view_desc, + Some(&mut output), + ) + .map_err(NewVideoProcessorError::CreateOutput)?; + output.unwrap() + }; + + texture_desc.Width = input_size.Width as u32; + texture_desc.Height = input_size.Height as u32; + texture_desc.Format = input_format; + texture_desc.BindFlags = (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32; + let video_input_texture = unsafe { + let mut texture = None; + d3d_device + .CreateTexture2D(&texture_desc, None, Some(&mut texture)) + .map_err(NewVideoProcessorError::CreateInput)?; + texture.unwrap() + }; + + let input_view_desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { + ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPIV { + MipSlice: 0, + ..Default::default() + }, + }, + ..Default::default() + }; + let video_input = unsafe { + let mut input = None; + video_device + .CreateVideoProcessorInputView( + &video_input_texture, + &video_enum, + &input_view_desc, + Some(&mut input), + ) + .map_err(NewVideoProcessorError::CreateInput)?; + input.unwrap() + }; + + Ok(Self { + _d3d_device: d3d_device, + d3d_context, + + _video_device: video_device, + video_context, + video_processor, + video_output_texture, + video_output, + video_input_texture, + video_input, + }) + } + + pub fn output_texture(&self) -> &ID3D11Texture2D { + &self.video_output_texture + } + + pub fn process_texture( + &mut self, + input_texture: &ID3D11Texture2D, + ) -> windows::core::Result<()> { + // The caller is responsible for making sure they give us a + // texture that matches the input size we were initialized with. + + unsafe { + // Copy the texture to the video input texture + self.d3d_context + .CopyResource(&self.video_input_texture, input_texture); + + // Convert to NV12 + let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + OutputIndex: 0, + InputFrameOrField: 0, + pInputSurface: ManuallyDrop::new(Some(self.video_input.clone())), + ..Default::default() + }; + self.video_context.VideoProcessorBlt( + &self.video_processor, + &self.video_output, + 0, + &[video_stream], + ) + } + } +} + +fn compute_scale_factor(output_size: Vector2, input_size: Vector2) -> f32 { + let output_ratio = output_size.X / output_size.Y; + let input_ratio = input_size.X / input_size.Y; + + let mut scale_factor = output_size.X / input_size.X; + if output_ratio > input_ratio { + scale_factor = output_size.Y / input_size.Y; + } + + scale_factor +} + +fn compute_dest_rect(output_size: &SizeInt32, input_size: &SizeInt32) -> RectInt32 { + let scale = compute_scale_factor( + Vector2 { + X: output_size.Width as f32, + Y: output_size.Height as f32, + }, + Vector2 { + X: input_size.Width as f32, + Y: input_size.Height as f32, + }, + ); + let new_size = SizeInt32 { + Width: (input_size.Width as f32 * scale) as i32, + Height: (input_size.Height as f32 * scale) as i32, + }; + let mut offset_x = 0; + let mut offset_y = 0; + if new_size.Width != output_size.Width { + offset_x = (output_size.Width - new_size.Width) / 2; + } + if new_size.Height != output_size.Height { + offset_y = (output_size.Height - new_size.Height) / 2; + } + RectInt32 { + X: offset_x, + Y: offset_y, + Width: new_size.Width, + Height: new_size.Height, + } +} diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index e913a45613..3e55368042 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -10,7 +10,8 @@ cap-rendering = { path = "../rendering" } cap-editor = { path = "../editor" } cap-media = { path = "../media" } cap-flags = { path = "../flags" } -cap-media-encoders = { path = "../media-encoders" } +cap-enc-ffmpeg = { path = "../enc-ffmpeg" } +cap-enc-gif = { path = "../enc-gif" } cap-media-info = { path = "../media-info" } tokio.workspace = true @@ -24,6 +25,9 @@ serde = { workspace = true } specta.workspace = true serde_json = "1.0.140" tracing.workspace = true +gifski = "1.32" +imgref = "1.10" +rgb = "0.8" [dev-dependencies] clap = { version = "4.5.41", features = ["derive"] } diff --git a/crates/export/src/gif.rs b/crates/export/src/gif.rs index 8d31b78f9a..1db7b287d7 100644 --- a/crates/export/src/gif.rs +++ b/crates/export/src/gif.rs @@ -1,11 +1,9 @@ -use std::path::PathBuf; - -use cap_media_encoders::{GifEncoderWrapper, GifQuality as EncoderGifQuality}; use cap_project::XY; use cap_rendering::{ProjectUniforms, RenderSegment, RenderedFrame}; use futures::FutureExt; use serde::Deserialize; use specta::Type; +use std::path::PathBuf; use tracing::trace; use crate::{ExportError, ExporterBase}; @@ -69,13 +67,13 @@ impl GifExportSettings { // Create GIF encoder with quality settings let quality = self .quality - .map(|q| EncoderGifQuality { + .map(|q| cap_enc_gif::GifQuality { quality: q.quality.unwrap_or(90), fast: q.fast.unwrap_or(false), }) .unwrap_or_default(); - let mut gif_encoder = GifEncoderWrapper::new_with_quality( + let mut gif_encoder = cap_enc_gif::GifEncoderWrapper::new_with_quality( &gif_output_path, output_size.0, output_size.1, diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index f18ce7fac4..3b744dfd3a 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -1,8 +1,6 @@ -use std::{path::PathBuf, time::Duration}; - use crate::ExporterBase; use cap_editor::{AudioRenderer, get_audio_segments}; -use cap_media_encoders::{AACEncoder, AudioEncoder, H264Encoder, MP4File, MP4Input}; +use cap_enc_ffmpeg::{AACEncoder, AudioEncoder, H264Encoder, MP4File, MP4Input}; use cap_media_info::{RawVideoFormat, VideoInfo}; use cap_project::XY; use cap_rendering::{ProjectUniforms, RenderSegment, RenderedFrame}; @@ -10,6 +8,7 @@ use futures::FutureExt; use image::ImageBuffer; use serde::Deserialize; use specta::Type; +use std::{path::PathBuf, time::Duration}; use tracing::{info, trace, warn}; #[derive(Deserialize, Type, Clone, Copy, Debug)] diff --git a/crates/ffmpeg-utils/Cargo.toml b/crates/ffmpeg-utils/Cargo.toml new file mode 100644 index 0000000000..b1225b5f2d --- /dev/null +++ b/crates/ffmpeg-utils/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "cap-ffmpeg-utils" +version = "0.1.0" +edition = "2024" + +[dependencies] +ffmpeg.workspace = true + +[lints] +workspace = true diff --git a/crates/ffmpeg-utils/src/lib.rs b/crates/ffmpeg-utils/src/lib.rs new file mode 100644 index 0000000000..00a2b04050 --- /dev/null +++ b/crates/ffmpeg-utils/src/lib.rs @@ -0,0 +1,34 @@ +pub trait PlanarData { + fn plane_data(&self, index: usize) -> &[u8]; + fn plane_data_mut(&mut self, index: usize) -> &mut [u8]; +} + +impl PlanarData for ffmpeg::frame::Audio { + #[inline] + fn plane_data(&self, index: usize) -> &[u8] { + if index >= self.planes() { + panic!("out of bounds"); + } + + unsafe { + std::slice::from_raw_parts( + (*self.as_ptr()).data[index], + (*self.as_ptr()).linesize[0] as usize, + ) + } + } + + #[inline] + fn plane_data_mut(&mut self, index: usize) -> &mut [u8] { + if index >= self.planes() { + panic!("out of bounds"); + } + + unsafe { + std::slice::from_raw_parts_mut( + (*self.as_mut_ptr()).data[index], + (*self.as_ptr()).linesize[0] as usize, + ) + } + } +} diff --git a/crates/media-encoders/README.md b/crates/media-encoders/README.md deleted file mode 100644 index a86016d670..0000000000 --- a/crates/media-encoders/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# cap-media-encoders - -A comprehensive collection of media encoders for various output formats, designed to work seamlessly with the Cap media processing pipeline. The crate provides unified interfaces for encoding captured audio and video data into different formats using multiple underlying media frameworks. - -## Overview - -This crate serves as a unified interface for encoding captured media into various output formats, with FFmpeg providing cross-platform support and AVFoundation offering optimized macOS-specific encoding. All encoders support real-time frame queuing and proper finalization for streaming applications. - -## Available Encoders - -### FFmpeg-based Encoders - -**Video Encoders:** -- `H264Encoder` - H.264/AVC video encoding with configurable quality presets and bitrate control -- `MP4File` - Complete MP4 container handling H.264 video + audio stream muxing - -**Audio Encoders:** -- `AACEncoder` - AAC audio encoding at 320kbps with automatic resampling and format conversion -- `OpusEncoder` - Opus audio encoding at 128kbps optimized for voice and music -- `OggFile` - Ogg container wrapper specifically designed for Opus audio streams - -### AVFoundation-based Encoders (macOS only) - -**Video + Audio:** -- `MP4AVAssetWriterEncoder` - Native macOS encoder using AVAssetWriter for hardware-accelerated MP4 encoding - - Hardware acceleration support - - Pause/resume functionality for recording sessions - - Real-time encoding optimizations - -### Standalone Encoders - -**Animation:** -- `GifEncoderWrapper` - GIF animation encoder with advanced features: - - Floyd-Steinberg dithering for quality color reduction - - Custom 256-color palette with grayscale fallback - - Configurable frame delay and infinite loop support - -## Common Interface - -### AudioEncoder Trait - -All audio encoders implement the `AudioEncoder` trait providing: -- `queue_frame()` - Queue audio frames for encoding -- `finish()` - Finalize encoding and flush remaining data -- `boxed()` - Convert to boxed trait object for dynamic dispatch - -## Key Features - -### Format Handling -- **Automatic conversion**: Seamless pixel format and sample format conversion between input and encoder requirements -- **Resampling**: Built-in audio resampling for sample rate and format mismatches -- **Validation**: Input validation with detailed error reporting - -### Performance Optimizations -- **Hardware acceleration**: AVFoundation encoder leverages macOS VideoToolbox -- **Threading**: Multi-threaded encoding support where available -- **Real-time processing**: Optimized for live capture and streaming scenarios - -### Quality Control -- **Configurable presets**: H.264 encoding supports Slow, Medium, and Ultrafast presets -- **Bitrate control**: Intelligent bitrate calculation based on resolution and frame rate -- **Format-specific optimizations**: Each encoder tuned for its target format characteristics - -## Dependencies - -- `cap-media-info` - Media stream information structures -- `ffmpeg` - Cross-platform media processing (video and audio encoding) -- `gif` - GIF image format encoding -- `cidre` (macOS only) - AVFoundation bindings for native encoding - -## Integration - -The crate integrates with: -- **cap-media-info** for media stream configuration and format definitions -- **FFmpeg ecosystem** for broad codec and container support -- **macOS AVFoundation** for optimized native encoding on Apple platforms -- **Raw media pipelines** for direct frame processing - -## Error Handling - -Comprehensive error types for each encoder: -- `H264EncoderError` - Video encoding failures -- `AACEncoderError` / `OpusEncoderError` - Audio encoding issues -- `GifEncodingError` - Animation encoding problems -- `InitError` - Encoder initialization failures - -Each error type provides detailed context about failures including codec availability, format compatibility, and resource constraints. - -## Use Cases - -This crate is designed for applications requiring: -- **Screen recording** with multiple output formats -- **Live streaming** with real-time encoding -- **Video conferencing** with adaptive quality -- **Media conversion** between different formats -- **Animation creation** from frame sequences - -The modular design allows applications to choose the most appropriate encoder for their specific requirements, whether prioritizing quality, performance, or compatibility. \ No newline at end of file diff --git a/crates/media-encoders/src/lib.rs b/crates/media-encoders/src/lib.rs deleted file mode 100644 index 46107522d4..0000000000 --- a/crates/media-encoders/src/lib.rs +++ /dev/null @@ -1,23 +0,0 @@ -mod aac; -pub use aac::*; - -mod audio_encoder; -pub use audio_encoder::*; - -mod gif; -pub use gif::*; - -mod h264; -pub use h264::*; - -mod mp4; -#[allow(ambiguous_glob_reexports)] -pub use mp4::*; - -mod opus; -pub use opus::*; - -#[cfg(target_os = "macos")] -mod mp4_avassetwriter; -#[cfg(target_os = "macos")] -pub use mp4_avassetwriter::*; diff --git a/crates/media-info/Cargo.toml b/crates/media-info/Cargo.toml index 0d6b894c7e..f16081663a 100644 --- a/crates/media-info/Cargo.toml +++ b/crates/media-info/Cargo.toml @@ -8,5 +8,7 @@ ffmpeg.workspace = true thiserror.workspace = true cpal.workspace = true +cap-ffmpeg-utils = { path = "../ffmpeg-utils" } + [lints] workspace = true diff --git a/crates/media-info/src/lib.rs b/crates/media-info/src/lib.rs index 3c48a39de6..9ea1d1b1f0 100644 --- a/crates/media-info/src/lib.rs +++ b/crates/media-info/src/lib.rs @@ -1,3 +1,4 @@ +use cap_ffmpeg_utils::*; use cpal::{SampleFormat, SupportedBufferSize, SupportedStreamConfig}; use ffmpeg::frame; pub use ffmpeg::{ @@ -264,43 +265,3 @@ pub fn ffmpeg_sample_format_for(sample_format: SampleFormat) -> Option { _ => None, } } - -pub trait PlanarData { - fn plane_data(&self, index: usize) -> &[u8]; - - fn plane_data_mut(&mut self, index: usize) -> &mut [u8]; -} - -// The ffmpeg crate's implementation of the `data_mut` function is wrong for audio; -// per [the FFmpeg docs](https://www.ffmpeg.org/doxygen/7.0/structAVFrame.html]) only -// the linesize of the first plane may be set for planar audio, and so we need to use -// that linesize for the rest of the planes (else they will appear to be empty slices). -impl PlanarData for frame::Audio { - #[inline] - fn plane_data(&self, index: usize) -> &[u8] { - if index >= self.planes() { - panic!("out of bounds"); - } - - unsafe { - std::slice::from_raw_parts( - (*self.as_ptr()).data[index], - (*self.as_ptr()).linesize[0] as usize, - ) - } - } - - #[inline] - fn plane_data_mut(&mut self, index: usize) -> &mut [u8] { - if index >= self.planes() { - panic!("out of bounds"); - } - - unsafe { - std::slice::from_raw_parts_mut( - (*self.as_mut_ptr()).data[index], - (*self.as_ptr()).linesize[0] as usize, - ) - } - } -} diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 99728c5224..389e141b82 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -7,68 +7,10 @@ edition = "2024" [lints] workspace = true -[features] -default = [] -debug-logging = [] # Feature flag to control debug logging - [dependencies] -cap-project = { path = "../project" } -cap-flags = { path = "../flags" } -cap-audio = { path = "../audio" } -cap-camera = { path = "../camera", features = ["specta", "serde"] } -cap-camera-ffmpeg = { path = "../camera-ffmpeg" } -cap-cpal-ffmpeg = { path = "../cpal-ffmpeg" } -cap-fail = { path = "../fail" } -cap-media-encoders = { path = "../media-encoders" } cap-media-info = { path = "../media-info" } -cpal.workspace = true ffmpeg.workspace = true -flume.workspace = true -indexmap = "2.5.0" -num-traits = "0.2.19" -ringbuf = "0.4.7" -serde = { workspace = true } -specta.workspace = true -tempfile = "3.12.0" thiserror.workspace = true -tracing = { workspace = true } -futures = { workspace = true } -axum = { version = "0.7.9", features = ["macros", "ws"] } -tokio.workspace = true -image = { version = "0.25.2", features = ["gif"] } -gif = "0.13.1" -tokio-util = "0.7.15" -kameo = "0.17.2" -scap = { workspace = true } -sync_wrapper = "1.0.2" -scap-ffmpeg = { path = "../scap-ffmpeg" } -scap-targets = { path = "../scap-targets" } - -[target.'cfg(target_os = "macos")'.dependencies] -cidre = { workspace = true } -cocoa = "0.26.0" -core-graphics = "0.24.0" -core-foundation = "0.10.0" -objc = "0.2.7" -objc-foundation = "0.1.1" -objc2-foundation = { version = "0.2.2", features = ["NSValue"] } -screencapturekit = "0.3.5" -scap-screencapturekit = { path = "../scap-screencapturekit" } - -[target.'cfg(target_os = "windows")'.dependencies] -windows = { workspace = true, features = [ - "Win32_Foundation", - "Win32_System", - "Win32_System_Threading", - "Win32_Graphics_Gdi", - "Win32_Graphics_Dwm", - "Win32_UI_WindowsAndMessaging", - "Win32_UI_HiDpi", - "Win32_Media_MediaFoundation", -] } -windows-capture = { workspace = true } -cap-camera-windows = { path = "../camera-windows" } -scap-direct3d = { path = "../scap-direct3d" } [dev-dependencies] inquire = "0.7.5" diff --git a/crates/mediafoundation-ffmpeg/Cargo.toml b/crates/mediafoundation-ffmpeg/Cargo.toml new file mode 100644 index 0000000000..28a844b406 --- /dev/null +++ b/crates/mediafoundation-ffmpeg/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "cap-mediafoundation-ffmpeg" +version = "0.1.0" +edition = "2024" + +[dependencies] +ffmpeg = { workspace = true } +cap-media-info = { path = "../media-info" } +tracing = { workspace = true } +flume = { workspace = true } +thiserror = { workspace = true } + +[target.'cfg(windows)'.dependencies] +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } +windows = { workspace = true, features = [ + "System", + "Graphics_Capture", + "Graphics_DirectX_Direct3D11", + "Foundation_Metadata", + "Win32_Graphics_Gdi", + "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi_Common", + "Win32_Graphics_Dxgi", + "Win32_System_Com", + "Win32_System_Com_StructuredStorage", + "Win32_System_Ole", + "Win32_System_Threading", + "Win32_System_WinRT_Direct3D11", + "Win32_System_WinRT_Graphics_Capture", + "Win32_UI_WindowsAndMessaging", + "Win32_Media_MediaFoundation", + "Win32_Media_DxMediaObjects", + "Win32_Foundation", + "Foundation_Collections", + "Win32_System_Variant", + "Storage_Search", + "Storage_Streams", + "Win32_UI_Input", + "Media_Core", + "Media_MediaProperties", + "Media_Transcoding", + "Win32_Storage_FileSystem", + "Win32_System_Diagnostics_Debug", + "Win32_UI_Input_KeyboardAndMouse", + "Security_Authorization_AppCapabilityAccess", +] } +windows-numerics = "0.2.0" + +[lints] +workspace = true diff --git a/crates/mediafoundation-ffmpeg/README.md b/crates/mediafoundation-ffmpeg/README.md new file mode 100644 index 0000000000..d20c3c8e18 --- /dev/null +++ b/crates/mediafoundation-ffmpeg/README.md @@ -0,0 +1,110 @@ +# MediaFoundation-FFmpeg H264 Muxing + +This crate provides utilities for muxing H264 encoded samples from Windows MediaFoundation into container formats using FFmpeg. + +## Purpose + +When using MediaFoundation for hardware-accelerated H264 encoding, the encoded output needs to be muxed into a container format (MP4, MKV, etc.). This crate bridges MediaFoundation's `IMFSample` output with FFmpeg's powerful muxing capabilities. + +## Features + +- Extract H264 data from MediaFoundation `IMFSample` objects +- Mux H264 streams into various container formats via FFmpeg +- Automatic keyframe detection +- Proper timestamp handling (converts from MediaFoundation's 100ns units to microseconds) +- Support for multiple output formats (MP4, MKV, etc.) + +## Usage + +### Basic Example + +```rust +use cap_mediafoundation_ffmpeg::{H264SampleMuxer, MuxerConfig}; +use std::path::PathBuf; + +// Configure the muxer +let config = MuxerConfig { + width: 1920, + height: 1080, + fps: 30, + bitrate: 5_000_000, // 5 Mbps +}; + +// Create the muxer +let mut muxer = H264SampleMuxer::new_mp4( + PathBuf::from("output.mp4"), + config, +)?; + +// Write MediaFoundation samples +// (assuming you have an encoder producing IMFSample objects) +for sample in encoded_samples { + muxer.write_sample(&sample)?; +} + +// Finish muxing +muxer.finish()?; +``` + +### Using Raw H264 Data + +If you already have extracted H264 data: + +```rust +muxer.write_h264_data( + &h264_data, + pts, // presentation timestamp in microseconds + dts, // decode timestamp in microseconds + duration, // duration in microseconds + is_keyframe, +)?; +``` + +## Integration with MediaFoundation + +This crate is designed to work with MediaFoundation H264 encoders. After encoding a frame with MediaFoundation: + +1. The encoder produces an `IMFSample` containing H264 data +2. Pass the sample to `write_sample()` +3. The muxer extracts the H264 data and timing information +4. The data is muxed into the output container + +## Timestamp Handling + +MediaFoundation uses 100-nanosecond units for timestamps, while FFmpeg typically works with microseconds. This crate automatically handles the conversion: + +- MediaFoundation: 100ns units +- This crate's API: microseconds +- FFmpeg internal: time_base units (configured based on FPS) + +## Keyframe Detection + +The muxer automatically detects IDR frames (keyframes) by inspecting the H264 NAL unit types. This ensures proper seeking and playback in the output file. + +## Audio Support + +The crate also includes utility traits for converting FFmpeg audio frames to MediaFoundation samples: + +- `AudioExt`: Convert `ffmpeg::frame::Audio` to `IMFSample` +- `PlanarData`: Access planar audio data + +## Requirements + +- Windows (MediaFoundation is Windows-only) +- FFmpeg libraries +- A MediaFoundation H264 encoder (hardware or software) + +## Error Handling + +All operations return proper error types that can be handled: + +```rust +match muxer.write_sample(&sample) { + Ok(_) => // Success + Err(e) => eprintln!("Failed to write sample: {}", e), +} +``` + +## Thread Safety + +The `H264SampleMuxer` is not thread-safe and should be used from a single thread. If you need concurrent access, wrap it in appropriate synchronization primitives. \ No newline at end of file diff --git a/crates/mediafoundation-ffmpeg/examples/usage.rs b/crates/mediafoundation-ffmpeg/examples/usage.rs new file mode 100644 index 0000000000..f37ad81572 --- /dev/null +++ b/crates/mediafoundation-ffmpeg/examples/usage.rs @@ -0,0 +1,197 @@ +fn main() { + #[cfg(windows)] + win::main(); +} + +#[cfg(windows)] +mod win { + use cap_mediafoundation_ffmpeg::{H264StreamMuxer, MuxerConfig}; + use ffmpeg::format; + use std::path::PathBuf; + + /// Example of using H264StreamMuxer with an existing FFmpeg output context + /// This demonstrates how to integrate with MP4File or similar structures + fn example_with_shared_output() -> Result<(), Box> { + // Initialize FFmpeg + ffmpeg::init()?; + + // Create an output context (this would normally be owned by MP4File) + let output_path = PathBuf::from("output.mp4"); + let mut output = format::output(&output_path)?; + + // Configure the H264 muxer + let config = MuxerConfig { + width: 1920, + height: 1080, + fps: 30, + bitrate: 5_000_000, // 5 Mbps + }; + + // Add the H264 stream and create the muxer + // Note: We need to add the stream before writing the header + let mut h264_muxer = H264StreamMuxer::new(&mut output, config)?; + + // You might also have other streams (like audio) added to the same output + // ... add audio stream here if needed ... + + // Write the header after all streams are added + output.write_header()?; + + // Now you can write H264 samples from MediaFoundation + #[cfg(windows)] + { + // Example: Write samples from MediaFoundation + // let sample: IMFSample = get_sample_from_media_foundation(); + // h264_muxer.write_sample(&sample)?; + } + + // Or write raw H264 data + let example_h264_data = vec![0, 0, 0, 1, 0x65]; // Example keyframe NAL + h264_muxer.write_h264_data( + &example_h264_data, + 0, // pts in microseconds + 0, // dts in microseconds + 33333, // duration in microseconds (1/30 fps) + true, // is_keyframe + &mut output, + )?; + + // Finish the muxer (doesn't write trailer) + h264_muxer.finish()?; + + // Write the trailer (this would be done by MP4File::finish()) + output.write_trailer()?; + + Ok(()) + } + + /// Example showing how this would integrate with an MP4File-like structure + struct MP4FileExample { + output: format::context::Output, + h264_muxer: Option, + is_finished: bool, + } + + impl MP4FileExample { + fn new(output_path: PathBuf, video_config: MuxerConfig) -> Result { + let mut output = format::output(&output_path)?; + + // Add H264 stream and create muxer + let h264_muxer = H264StreamMuxer::new(&mut output, video_config)?; + + // You could add audio streams here too + // ... + + // Write header after all streams are added + output.write_header()?; + + Ok(Self { + output, + h264_muxer: Some(h264_muxer), + is_finished: false, + }) + } + + #[cfg(windows)] + fn write_sample( + &mut self, + sample: &windows::Win32::Media::MediaFoundation::IMFSample, + ) -> Result<(), Box> { + if let Some(muxer) = &mut self.h264_muxer { + muxer.write_sample(sample, &mut self.output)?; + } + Ok(()) + } + + fn write_h264_data( + &mut self, + data: &[u8], + pts: i64, + dts: i64, + duration: i64, + is_keyframe: bool, + ) -> Result<(), ffmpeg::Error> { + if let Some(muxer) = &mut self.h264_muxer { + muxer.write_h264_data(data, pts, dts, duration, is_keyframe, &mut self.output)?; + } + Ok(()) + } + + fn finish(&mut self) -> Result<(), ffmpeg::Error> { + if self.is_finished { + return Ok(()); + } + self.is_finished = true; + + if let Some(muxer) = &mut self.h264_muxer { + muxer.finish()?; + } + + // Write trailer + self.output.write_trailer()?; + Ok(()) + } + } + + /// Alternative approach using the owned version for standalone use + fn example_with_owned_muxer() -> Result<(), Box> { + use mediafoundation_ffmpeg::H264SampleMuxerOwned; + + // Initialize FFmpeg + ffmpeg::init()?; + + let config = MuxerConfig { + width: 1920, + height: 1080, + fps: 30, + bitrate: 5_000_000, + }; + + // Create a standalone muxer that owns its output + let mut muxer = + H264SampleMuxerOwned::new_mp4(PathBuf::from("standalone_output.mp4"), config)?; + + // Write some H264 data + let example_h264_data = vec![0, 0, 0, 1, 0x65]; // Example keyframe NAL + muxer.write_h264_data( + &example_h264_data, + 0, // pts + 0, // dts + 33333, // duration + true, // is_keyframe + )?; + + // The muxer automatically finishes and writes trailer when dropped + muxer.finish()?; + + Ok(()) + } + + fn main() -> Result<(), Box> { + println!("Example 1: Using H264StreamMuxer with shared output"); + example_with_shared_output()?; + + println!("\nExample 2: Using H264SampleMuxerOwned for standalone use"); + example_with_owned_muxer()?; + + println!("\nExample 3: Using MP4FileExample with integrated muxer"); + let mut mp4_file = MP4FileExample::new( + PathBuf::from("integrated_output.mp4"), + MuxerConfig { + width: 1920, + height: 1080, + fps: 30, + bitrate: 5_000_000, + }, + )?; + + // Write some test data + let example_h264_data = vec![0, 0, 0, 1, 0x65]; + mp4_file.write_h264_data(&example_h264_data, 0, 0, 33333, true)?; + + // Finish writing + mp4_file.finish()?; + + Ok(()) + } +} diff --git a/crates/mediafoundation-ffmpeg/src/audio.rs b/crates/mediafoundation-ffmpeg/src/audio.rs new file mode 100644 index 0000000000..444144b6b0 --- /dev/null +++ b/crates/mediafoundation-ffmpeg/src/audio.rs @@ -0,0 +1,46 @@ +use windows::Win32::Media::MediaFoundation::{IMFSample, MFCreateMemoryBuffer, MFCreateSample}; + +pub trait AudioExt { + fn to_sample(&self) -> windows::core::Result; +} + +impl AudioExt for ffmpeg::frame::Audio { + fn to_sample(&self) -> windows::core::Result { + let sample = unsafe { MFCreateSample()? }; + + let length = (self.samples() * self.format().bytes()) + * (if self.is_planar() { + self.channels() as usize + } else { + 1 + }); + let buffer = unsafe { MFCreateMemoryBuffer(length as u32)? }; + + unsafe { sample.AddBuffer(&buffer)? }; + + let mut buffer_ptr: *mut u8 = std::ptr::null_mut(); + unsafe { buffer.Lock(&mut buffer_ptr, None, None)? }; + + unsafe { + std::ptr::copy_nonoverlapping(self.data(0).as_ptr(), buffer_ptr, length as usize); + } + + unsafe { buffer.SetCurrentLength(length as u32)? } + + unsafe { buffer.Unlock()? }; + + if let Some(pts) = self.pts() { + unsafe { + sample.SetSampleTime((pts as f64 / self.rate() as f64 * 10_000_000_f64) as i64)? + }; + } + + unsafe { + sample.SetSampleDuration( + ((self.samples() as f64 / self.rate() as f64) * 10_000_000_f64) as i64, + )? + }; + + Ok(sample) + } +} diff --git a/crates/mediafoundation-ffmpeg/src/h264.rs b/crates/mediafoundation-ffmpeg/src/h264.rs new file mode 100644 index 0000000000..29968dc12c --- /dev/null +++ b/crates/mediafoundation-ffmpeg/src/h264.rs @@ -0,0 +1,180 @@ +use cap_mediafoundation_utils::*; +use ffmpeg::{Rational, ffi::av_rescale_q, packet}; +use tracing::info; +use windows::Win32::Media::MediaFoundation::{IMFSample, MFSampleExtension_CleanPoint}; + +/// Configuration for H264 muxing +#[derive(Clone, Debug)] +pub struct MuxerConfig { + pub width: u32, + pub height: u32, + pub fps: u32, + pub bitrate: u32, +} + +/// H264 stream muxer that works with external FFmpeg output contexts +/// This version doesn't hold a reference to the output, making it easier to integrate +pub struct H264StreamMuxer { + stream_index: usize, + time_base: ffmpeg::Rational, + is_finished: bool, + frame_count: u64, +} + +impl H264StreamMuxer { + /// Add an H264 stream to an output context and create a muxer for it + /// Returns the muxer which can be used to write packets to the stream + /// Note: The caller must call write_header() on the output after adding all streams + pub fn new( + output: &mut ffmpeg::format::context::Output, + config: MuxerConfig, + ) -> Result { + info!("Adding H264 stream to output context"); + + // Find H264 codec + let h264_codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::H264) + .ok_or(ffmpeg::Error::DecoderNotFound)?; + + // Add video stream + let mut stream = output.add_stream(h264_codec)?; + let stream_index = stream.index(); + + let time_base = ffmpeg::Rational::new(1, config.fps as i32 * 1000); + stream.set_time_base(time_base); + + // Configure stream parameters + unsafe { + let codecpar = (*stream.as_mut_ptr()).codecpar; + (*codecpar).codec_type = ffmpeg::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO; + (*codecpar).codec_id = ffmpeg::ffi::AVCodecID::AV_CODEC_ID_H264; + (*codecpar).width = config.width as i32; + (*codecpar).height = config.height as i32; + (*codecpar).bit_rate = config.bitrate as i64; + (*codecpar).format = ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NV12 as i32; + + // Set frame rate + (*stream.as_mut_ptr()).avg_frame_rate = ffmpeg::ffi::AVRational { + num: config.fps as i32, + den: 1, + }; + (*stream.as_mut_ptr()).r_frame_rate = ffmpeg::ffi::AVRational { + num: config.fps as i32, + den: 1, + }; + } + + info!( + "H264 stream added: {}x{} @ {} fps, {} kbps", + config.width, + config.height, + config.fps, + config.bitrate / 1000 + ); + + Ok(Self { + stream_index, + time_base, + is_finished: false, + frame_count: 0, + }) + } + + /// Write an H264 sample from MediaFoundation to the output + pub fn write_sample( + &mut self, + sample: &IMFSample, + output: &mut ffmpeg::format::context::Output, + ) -> Result<(), Box> { + if self.is_finished { + return Err("Muxer is already finished".into()); + } + + let mut packet = self.mf_sample_to_avpacket(sample)?; + + packet.rescale_ts( + self.time_base, + output.stream(self.stream_index).unwrap().time_base(), + ); + + packet.write_interleaved(output)?; + + Ok(()) + } + + fn mf_sample_to_avpacket(&self, sample: &IMFSample) -> windows::core::Result { + let len = unsafe { sample.GetTotalLength()? }; + let mut packet = ffmpeg::Packet::new(len as usize); + + { + let buffer = unsafe { sample.ConvertToContiguousBuffer()? }; + let data = buffer.lock()?; + + packet + .data_mut() + .unwrap() + .copy_from_slice(&data[0..len as usize]); + } + + let pts = unsafe { sample.GetSampleTime() } + .ok() + .map(|v| mf_from_mf_time(self.time_base, v)); + packet.set_pts(pts); + packet.set_dts(pts); + + let duration = unsafe { sample.GetSampleDuration() } + .ok() + .map(|v| mf_from_mf_time(self.time_base, v)) + .unwrap_or_default(); + packet.set_duration(duration); + + if let Ok(t) = unsafe { sample.GetUINT32(&MFSampleExtension_CleanPoint) } + && t != 0 + { + packet.set_flags(packet::Flags::KEY); + } + + packet.set_stream(self.stream_index); + + // if let Ok(decode_timestamp) = + // unsafe { sample.GetUINT64(&MFSampleExtension_DecodeTimestamp) } + // { + // packet.set_dts(Some(mf_from_mf_time( + // self.time_base, + // decode_timestamp as i64, + // ))); + // } + + Ok(packet) + } + + /// Mark the muxer as finished (note: does not write trailer, caller is responsible) + pub fn finish(&mut self) -> Result<(), ffmpeg::Error> { + if self.is_finished { + return Ok(()); + } + + self.is_finished = true; + + info!("Finishing H264 muxer, wrote {} frames", self.frame_count); + + // Note: Caller is responsible for writing trailer to the output context + + Ok(()) + } + + /// Get the number of frames written + pub fn frame_count(&self) -> u64 { + self.frame_count + } + + /// Check if the muxer is finished + pub fn is_finished(&self) -> bool { + self.is_finished + } +} + +const MF_TIMEBASE: ffmpeg::Rational = ffmpeg::Rational(1, 10_000_000); + +fn mf_from_mf_time(tb: Rational, stime: i64) -> i64 { + unsafe { av_rescale_q(stime, MF_TIMEBASE.into(), tb.into()) } +} diff --git a/crates/mediafoundation-ffmpeg/src/lib.rs b/crates/mediafoundation-ffmpeg/src/lib.rs new file mode 100644 index 0000000000..07fe6e79ec --- /dev/null +++ b/crates/mediafoundation-ffmpeg/src/lib.rs @@ -0,0 +1,7 @@ +#![cfg(windows)] + +mod audio; +mod h264; + +pub use audio::AudioExt; +pub use h264::{H264StreamMuxer, MuxerConfig}; diff --git a/crates/mediafoundation-utils/Cargo.toml b/crates/mediafoundation-utils/Cargo.toml new file mode 100644 index 0000000000..18da3b7a55 --- /dev/null +++ b/crates/mediafoundation-utils/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "cap-mediafoundation-utils" +version = "0.1.0" +edition = "2024" + +[dependencies] + +[target.'cfg(windows)'.dependencies] +windows-core = { workspace = true } +windows = { workspace = true, features = [ + "Win32_Media_MediaFoundation", + "Win32_Media_DirectShow", + "Win32_System_Com", +] } + +[lints] +workspace = true diff --git a/crates/mediafoundation-utils/src/lib.rs b/crates/mediafoundation-utils/src/lib.rs new file mode 100644 index 0000000000..5893a2e049 --- /dev/null +++ b/crates/mediafoundation-utils/src/lib.rs @@ -0,0 +1,66 @@ +#![cfg(windows)] + +use std::{ + ops::{Deref, DerefMut}, + ptr::null_mut, +}; +use windows::{ + Win32::{ + Media::MediaFoundation::{IMFMediaBuffer, MFSTARTUP_FULL, MFStartup}, + System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize}, + }, + core::Result, +}; + +// This is the value for Win7+ +pub const MF_VERSION: u32 = 131184; + +pub fn thread_init() { + let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; + let _ = unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) }; +} + +pub trait IMFMediaBufferExt { + fn lock(&self) -> Result>; +} + +impl IMFMediaBufferExt for IMFMediaBuffer { + fn lock(&self) -> Result> { + let mut bytes_ptr = null_mut(); + let mut size = 0; + + unsafe { + self.Lock(&mut bytes_ptr, None, Some(&mut size))?; + } + + Ok(IMFMediaBufferLock { + source: self, + bytes: unsafe { std::slice::from_raw_parts_mut(bytes_ptr, size as usize) }, + }) + } +} + +pub struct IMFMediaBufferLock<'a> { + source: &'a IMFMediaBuffer, + bytes: &'a mut [u8], +} + +impl<'a> Drop for IMFMediaBufferLock<'a> { + fn drop(&mut self) { + let _ = unsafe { self.source.Unlock() }; + } +} + +impl<'a> Deref for IMFMediaBufferLock<'a> { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + self.bytes + } +} + +impl<'a> DerefMut for IMFMediaBufferLock<'a> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.bytes + } +} diff --git a/crates/recording/Cargo.toml b/crates/recording/Cargo.toml index 95388d5611..52ac8672a5 100644 --- a/crates/recording/Cargo.toml +++ b/crates/recording/Cargo.toml @@ -15,11 +15,12 @@ cap-flags = { path = "../flags" } cap-utils = { path = "../utils" } scap-targets = { path = "../scap-targets" } cap-cursor-capture = { path = "../cursor-capture" } -cap-media-encoders = { path = "../media-encoders" } cap-media-info = { path = "../media-info" } cap-cursor-info = { path = "../cursor-info" } -cap-camera = { path = "../camera" } +cap-camera = { path = "../camera", features = ["serde", "specta"] } cap-camera-ffmpeg = { path = "../camera-ffmpeg" } +cap-enc-ffmpeg = { path = "../enc-ffmpeg" } +cap-ffmpeg-utils = { path = "../ffmpeg-utils" } specta.workspace = true tokio.workspace = true @@ -54,12 +55,16 @@ objc = "0.2.7" cidre = { workspace = true } objc2-app-kit = "0.3.1" scap-screencapturekit = { path = "../scap-screencapturekit" } +cap-enc-avfoundation = { path = "../enc-avfoundation" } [target.'cfg(target_os = "windows")'.dependencies] +cap-enc-mediafoundation = { path = "../enc-mediafoundation" } +cap-mediafoundation-ffmpeg = { path = "../mediafoundation-ffmpeg" } +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } windows = { workspace = true, features = [ - "Win32_Foundation", - "Win32_Graphics_Gdi", - "Win32_UI_WindowsAndMessaging", + "Win32_Foundation", + "Win32_Graphics_Gdi", + "Win32_UI_WindowsAndMessaging", ] } scap-direct3d = { path = "../scap-direct3d" } scap-ffmpeg = { path = "../scap-ffmpeg" } @@ -67,4 +72,4 @@ scap-cpal = { path = "../scap-cpal" } [dev-dependencies] tempfile = "3.20.0" -tracing-subscriber = "0.3.19" +tracing-subscriber = { version = "0.3.19", default-features = false } diff --git a/crates/recording/examples/recording-cli.rs b/crates/recording/examples/recording-cli.rs index be69e6082d..fc1c126b22 100644 --- a/crates/recording/examples/recording-cli.rs +++ b/crates/recording/examples/recording-cli.rs @@ -2,6 +2,7 @@ use std::time::Duration; use cap_recording::{RecordingBaseInputs, screen_capture::ScreenCaptureTarget}; use scap_targets::Display; +use tracing::info; #[tokio::main] pub async fn main() { @@ -19,7 +20,7 @@ pub async fn main() { let dir = tempfile::tempdir().unwrap(); - println!("Recording to directory '{}'", dir.path().display()); + info!("Recording to directory '{}'", dir.path().display()); let (handle, _ready_rx) = cap_recording::spawn_studio_recording_actor( "test".to_string(), @@ -28,18 +29,12 @@ pub async fn main() { capture_target: ScreenCaptureTarget::Display { id: Display::primary().id(), }, - // ScreenCaptureTarget::Window { - // id: Window::list() - // .into_iter() - // .find(|w| w.owner_name().unwrap_or_default().contains("Brave")) - // .unwrap() - // .id(), - // }, capture_system_audio: true, - mic_feed: &None, + camera_feed: None, + mic_feed: None, }, - None, - true, + false, + // true, ) .await .unwrap(); diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 25962d1563..30fc96f14e 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -1,14 +1,3 @@ -use cap_media::MediaError; - -use cap_media_info::AudioInfo; -use flume::{Receiver, Sender}; -use std::{ - future::Future, - path::PathBuf, - sync::{Arc, atomic::AtomicBool}, - time::SystemTime, -}; - use crate::{ RecordingError, feeds::microphone::MicrophoneFeedLock, @@ -18,6 +7,15 @@ use crate::{ ScreenCaptureTarget, screen_capture, }, }; +use cap_media::MediaError; +use cap_media_info::AudioInfo; +use flume::{Receiver, Sender}; +use std::{ + future::Future, + path::PathBuf, + sync::{Arc, atomic::AtomicBool}, + time::SystemTime, +}; pub trait MakeCapturePipeline: ScreenCaptureFormat + std::fmt::Debug + 'static { fn make_studio_mode_pipeline( @@ -59,7 +57,7 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { let screen_config = source.0.info(); tracing::info!("screen config: {:?}", screen_config); - let mut screen_encoder = cap_media_encoders::MP4AVAssetWriterEncoder::init( + let mut screen_encoder = cap_enc_avfoundation::MP4Encoder::init( "screen", screen_config, None, @@ -134,7 +132,7 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { let has_audio_sources = audio_mixer.has_sources(); let mp4 = Arc::new(std::sync::Mutex::new( - cap_media_encoders::MP4AVAssetWriterEncoder::init( + cap_enc_avfoundation::MP4Encoder::init( "mp4", source.0.info(), has_audio_sources.then_some(AudioMixer::info()), @@ -225,7 +223,7 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { } #[cfg(windows)] -impl MakeCapturePipeline for screen_capture::AVFrameCapture { +impl MakeCapturePipeline for screen_capture::Direct3DCapture { fn make_studio_mode_pipeline( mut builder: PipelineBuilder, source: ( @@ -237,33 +235,143 @@ impl MakeCapturePipeline for screen_capture::AVFrameCapture { where Self: Sized, { - use cap_media_encoders::{H264Encoder, MP4File}; + use windows::Graphics::SizeInt32; + + cap_mediafoundation_utils::thread_init(); let screen_config = source.0.info(); - let mut screen_encoder = MP4File::init( - "screen", - output_path, - |o| H264Encoder::builder("screen", dbg!(screen_config)).build(o), - |_| None, - ) - .map_err(|e| MediaError::Any(e.to_string().into()))?; + + let mut output = ffmpeg::format::output(&output_path) + .map_err(|e| MediaError::Any(format!("CreateOutput: {e}").into()))?; + + let screen_encoder = { + let native_encoder = cap_enc_mediafoundation::H264Encoder::new( + source.0.d3d_device(), + screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(), + SizeInt32 { + Width: screen_config.width as i32, + Height: screen_config.height as i32, + }, + source.0.config().fps(), + 0.1, + ); + + match native_encoder { + Ok(encoder) => { + let mut muxer = cap_mediafoundation_ffmpeg::H264StreamMuxer::new( + &mut output, + cap_mediafoundation_ffmpeg::MuxerConfig { + width: screen_config.width, + height: screen_config.height, + fps: screen_config.fps(), + bitrate: encoder.bitrate(), + }, + ) + .map_err(|e| MediaError::Any(format!("NativeH264/{e}").into()))?; + + encoder + .start() + .map_err(|e| MediaError::Any(format!("ScreenEncoderStart: {e}").into()))?; + + either::Left((encoder, muxer)) + } + Err(e) => { + use tracing::{error, info}; + + error!("Failed to create native encoder: {e}"); + info!("Falling back to software H264 encoder"); + + either::Right( + cap_enc_ffmpeg::H264Encoder::builder("screen", screen_config) + .build(&mut output) + .map_err(|e| MediaError::Any(format!("H264Encoder/{e}").into()))?, + ) + } + } + }; + + output + .write_header() + .map_err(|e| MediaError::Any(format!("OutputHeader/{e}").into()))?; builder.spawn_source("screen_capture", source.0); let (timestamp_tx, timestamp_rx) = flume::bounded(1); builder.spawn_task("screen_capture_encoder", move |ready| { - let mut timestamp_tx = Some(timestamp_tx); - let _ = ready.send(Ok(())); + match screen_encoder { + either::Left((mut encoder, mut muxer)) => { + use windows::Win32::Media::MediaFoundation; + + cap_mediafoundation_utils::thread_init(); + + let _ = ready.send(Ok(())); + + let mut timestamp_tx = Some(timestamp_tx); + + while let Ok(e) = encoder.get_event() { + match e { + MediaFoundation::METransformNeedInput => { + let Ok((frame, timestamp)) = source.1.recv() else { + break; + }; + + if let Some(timestamp_tx) = timestamp_tx.take() { + timestamp_tx.send(timestamp).unwrap(); + } + + let frame_time = frame + .inner() + .SystemRelativeTime() + .map_err(|e| format!("FrameTime: {e}"))?; + + encoder + .handle_needs_input(frame.texture(), frame_time) + .map_err(|e| format!("NeedsInput: {e}"))?; + } + MediaFoundation::METransformHaveOutput => { + if let Some(output_sample) = encoder + .handle_has_output() + .map_err(|e| format!("HasOutput: {e}"))? + { + muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| format!("WriteSample: {e}"))?; + } + } + _ => {} + } + } - while let Ok(frame) = source.1.recv() { - if let Some(timestamp_tx) = timestamp_tx.take() { - timestamp_tx.send(frame.1).unwrap(); + encoder + .finish() + .map_err(|e| format!("EncoderFinish: {e}"))?; + } + either::Right(mut encoder) => { + let mut timestamp_tx = Some(timestamp_tx); + let _ = ready.send(Ok(())); + + while let Ok((frame, timestamp)) = source.1.recv() { + use scap_ffmpeg::AsFFmpeg; + + if let Some(timestamp_tx) = timestamp_tx.take() { + let _ = timestamp_tx.send(timestamp); + } + + let ff_frame = frame + .as_ffmpeg() + .map_err(|e| format!("FrameAsFfmpeg: {e}"))?; + + encoder.queue_frame(ff_frame, &mut output); + } + encoder.finish(&mut output); } - // dbg!(frame.1); - screen_encoder.queue_video_frame(frame.0); } - screen_encoder.finish(); + + output + .write_trailer() + .map_err(|e| format!("WriteTrailer: {e}"))?; + Ok(()) }); @@ -284,7 +392,10 @@ impl MakeCapturePipeline for screen_capture::AVFrameCapture { where Self: Sized, { - use cap_media_encoders::{AACEncoder, AudioEncoder, H264Encoder, MP4File}; + use cap_enc_ffmpeg::{AACEncoder, AudioEncoder}; + use windows::Graphics::SizeInt32; + + cap_mediafoundation_utils::thread_init(); let (audio_tx, audio_rx) = flume::bounded(64); let mut audio_mixer = AudioMixer::new(audio_tx); @@ -301,33 +412,86 @@ impl MakeCapturePipeline for screen_capture::AVFrameCapture { } let has_audio_sources = audio_mixer.has_sources(); - let screen_config = source.0.info(); - let mp4 = Arc::new(std::sync::Mutex::new( - MP4File::init( - "screen", - output_path, - |o| H264Encoder::builder("screen", screen_config).build(o), - |o| { - has_audio_sources.then(|| { - AACEncoder::init("mic_audio", AudioMixer::info(), o) - .map(|v| v.boxed()) - .map_err(Into::into) - }) + + let mut output = ffmpeg::format::output(&output_path) + .map_err(|e| MediaError::Any(format!("CreateOutput: {e}").into()))?; + + let screen_encoder = { + let native_encoder = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( + source.0.d3d_device(), + screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(), + SizeInt32 { + Width: screen_config.width as i32, + Height: screen_config.height as i32, }, - ) - .map_err(|e| MediaError::Any(e.to_string().into()))?, - )); + SizeInt32 { + Width: screen_config.width as i32, + Height: screen_config.height as i32, + }, + 30, + 0.15, + ); + + match native_encoder { + Ok(screen_encoder) => { + let screen_muxer = cap_mediafoundation_ffmpeg::H264StreamMuxer::new( + &mut output, + cap_mediafoundation_ffmpeg::MuxerConfig { + width: screen_config.width, + height: screen_config.height, + fps: 30, + bitrate: screen_encoder.bitrate(), + }, + ) + .map_err(|e| MediaError::Any(format!("NativeH264Muxer/{e}").into()))?; + + screen_encoder + .start() + .map_err(|e| MediaError::Any(format!("StartScreenEncoder/{e}").into()))?; + + either::Left((screen_encoder, screen_muxer)) + } + Err(e) => { + use tracing::{error, info}; - if has_audio_sources { + error!("Failed to create native encoder: {e}"); + info!("Falling back to software H264 encoder"); + + either::Right( + cap_enc_ffmpeg::H264Encoder::builder("screen", screen_config) + .build(&mut output) + .map_err(|e| MediaError::Any(format!("H264Encoder/{e}").into()))?, + ) + } + } + }; + + let audio_encoder = has_audio_sources + .then(|| { + AACEncoder::init("mic_audio", AudioMixer::info(), &mut output) + .map(|v| v.boxed()) + .map_err(|e| MediaError::Any(e.to_string().into())) + }) + .transpose() + .map_err(|e| MediaError::Any(format!("AACEncoder/{e}").into()))?; + + output + .write_header() + .map_err(|e| MediaError::Any(format!("OutputHeader/{e}").into()))?; + + let output = Arc::new(std::sync::Mutex::new(output)); + + if let Some(mut audio_encoder) = audio_encoder { builder.spawn_source("audio_mixer", audio_mixer); - let mp4 = mp4.clone(); + // let is_done = is_done.clone(); + let output = output.clone(); builder.spawn_task("audio_encoding", move |ready| { let _ = ready.send(Ok(())); while let Ok(frame) = audio_rx.recv() { - if let Ok(mut mp4) = mp4.lock() { - mp4.queue_audio_frame(frame); + if let Ok(mut output) = output.lock() { + audio_encoder.queue_frame(frame, &mut *output); } } Ok(()) @@ -337,21 +501,84 @@ impl MakeCapturePipeline for screen_capture::AVFrameCapture { builder.spawn_source("screen_capture", source.0); builder.spawn_task("screen_encoder", move |ready| { - let _ = ready.send(Ok(())); - while let Ok((frame, _unix_time)) = source.1.recv() { - if let Ok(mut mp4) = mp4.lock() { - // if pause_flag.load(std::sync::atomic::Ordering::Relaxed) { - // mp4.pause(); - // } else { - // mp4.resume(); - // } + match screen_encoder { + either::Left((mut encoder, mut muxer)) => { + use windows::Win32::Media::MediaFoundation; + + cap_mediafoundation_utils::thread_init(); + + let _ = ready.send(Ok(())); + + while let Ok(e) = encoder.get_event() { + match e { + MediaFoundation::METransformNeedInput => { + let Ok((frame, _)) = source.1.recv() else { + break; + }; + + let frame_time = frame + .inner() + .SystemRelativeTime() + .map_err(|e| format!("Frame Time: {e}"))?; + + encoder + .handle_needs_input(frame.texture(), frame_time) + .map_err(|e| format!("NeedsInput: {e}"))?; + } + MediaFoundation::METransformHaveOutput => { + if let Some(output_sample) = encoder + .handle_has_output() + .map_err(|e| format!("HasOutput: {e}"))? + { + let mut output = output.lock().unwrap(); + + muxer + .write_sample(&output_sample, &mut *output) + .map_err(|e| format!("WriteSample: {e}"))?; + } + } + _ => {} + } + } - mp4.queue_video_frame(frame); + encoder + .finish() + .map_err(|e| format!("EncoderFinish: {e}"))?; + } + either::Right(mut encoder) => { + let output = output.clone(); + + let _ = ready.send(Ok(())); + + while let Ok((frame, _unix_time)) = source.1.recv() { + let Ok(mut output) = output.lock() else { + continue; + }; + + // if pause_flag.load(std::sync::atomic::Ordering::Relaxed) { + // mp4.pause(); + // } else { + // mp4.resume(); + // } + + use scap_ffmpeg::AsFFmpeg; + + encoder.queue_frame( + frame + .as_ffmpeg() + .map_err(|e| format!("FrameAsFFmpeg: {e}"))?, + &mut output, + ); + } } } - if let Ok(mut mp4) = mp4.lock() { - mp4.finish(); - } + + output + .lock() + .map_err(|e| format!("OutputLock: {e}"))? + .write_trailer() + .map_err(|e| format!("WriteTrailer: {e}"))?; + Ok(()) }); @@ -368,7 +595,7 @@ type ScreenCaptureReturn = ( pub type ScreenCaptureMethod = screen_capture::CMSampleBufferCapture; #[cfg(windows)] -pub type ScreenCaptureMethod = screen_capture::AVFrameCapture; +pub type ScreenCaptureMethod = screen_capture::Direct3DCapture; pub async fn create_screen_capture( capture_target: &ScreenCaptureTarget, @@ -376,6 +603,7 @@ pub async fn create_screen_capture( max_fps: u32, audio_tx: Option>, start_time: SystemTime, + #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, ) -> Result, RecordingError> { let (video_tx, video_rx) = flume::bounded(16); @@ -387,8 +615,70 @@ pub async fn create_screen_capture( audio_tx, start_time, tokio::runtime::Handle::current(), + #[cfg(windows)] + d3d_device, ) .await .map(|v| (v, video_rx)) .map_err(|e| RecordingError::Media(MediaError::TaskLaunch(e.to_string()))) } + +#[cfg(windows)] +pub fn create_d3d_device() +-> windows::core::Result { + use windows::Win32::Graphics::{ + Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE}, + Direct3D11::{D3D11_CREATE_DEVICE_FLAG, ID3D11Device}, + }; + + let mut device = None; + let flags = { + use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_BGRA_SUPPORT; + + let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + if cfg!(feature = "d3ddebug") { + use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_DEBUG; + + flags |= D3D11_CREATE_DEVICE_DEBUG; + } + flags + }; + let mut result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, flags, &mut device); + if let Err(error) = &result { + use windows::Win32::Graphics::Dxgi::DXGI_ERROR_UNSUPPORTED; + + if error.code() == DXGI_ERROR_UNSUPPORTED { + use windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_WARP; + + result = create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device); + } + } + result?; + + fn create_d3d_device_with_type( + driver_type: D3D_DRIVER_TYPE, + flags: D3D11_CREATE_DEVICE_FLAG, + device: *mut Option, + ) -> windows::core::Result<()> { + unsafe { + use windows::Win32::{ + Foundation::HMODULE, + Graphics::Direct3D11::{D3D11_SDK_VERSION, D3D11CreateDevice}, + }; + + D3D11CreateDevice( + None, + driver_type, + HMODULE(std::ptr::null_mut()), + flags, + None, + D3D11_SDK_VERSION, + Some(device), + None, + None, + ) + } + } + + Ok(device.unwrap()) +} diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index 86399e6998..bcd10c60f5 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -186,8 +186,20 @@ pub async fn spawn_instant_recording_actor( (None, None) }; - let (screen_source, screen_rx) = - create_screen_capture(&inputs.capture_target, true, 30, system_audio.0, start_time).await?; + #[cfg(windows)] + let d3d_device = crate::capture_pipeline::create_d3d_device() + .map_err(|e| MediaError::Any(format!("CreateD3DDevice: {e}").into()))?; + + let (screen_source, screen_rx) = create_screen_capture( + &inputs.capture_target, + true, + 30, + system_audio.0, + start_time, + #[cfg(windows)] + d3d_device, + ) + .await?; debug!("screen capture: {screen_source:#?}"); diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index 2ed245ed6e..007599dda2 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -6,6 +6,9 @@ pub mod pipeline; pub mod sources; pub mod studio_recording; +pub use instant_recording::{ + CompletedInstantRecording, InstantRecordingActor, spawn_instant_recording_actor, +}; pub use sources::{camera, screen_capture}; pub use studio_recording::{ CompletedStudioRecording, StudioRecordingHandle, spawn_studio_recording_actor, diff --git a/crates/recording/src/pipeline/audio_buffer.rs b/crates/recording/src/pipeline/audio_buffer.rs index 9b85ced02e..39045e5cb5 100644 --- a/crates/recording/src/pipeline/audio_buffer.rs +++ b/crates/recording/src/pipeline/audio_buffer.rs @@ -1,6 +1,6 @@ use cap_audio::cast_bytes_to_f32_slice; - -use cap_media_info::{AudioInfo, PlanarData}; +use cap_ffmpeg_utils::*; +use cap_media_info::AudioInfo; use ffmpeg::encoder; pub use ffmpeg::util::frame::Audio as FFAudio; use std::collections::VecDeque; diff --git a/crates/recording/src/pipeline/control.rs b/crates/recording/src/pipeline/control.rs index 8fabc57d44..5df77bafac 100644 --- a/crates/recording/src/pipeline/control.rs +++ b/crates/recording/src/pipeline/control.rs @@ -10,8 +10,8 @@ pub enum Control { #[derive(Clone)] pub struct PipelineControlSignal { - last_value: Option, - receiver: Receiver, + pub last_value: Option, + pub receiver: Receiver, } impl PipelineControlSignal { diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index c9838f9706..cca201792b 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -1,4 +1,5 @@ use super::*; +use cap_ffmpeg_utils::PlanarData; use cidre::*; use kameo::prelude::*; @@ -93,8 +94,6 @@ impl Message for FrameHandler { frame.set_rate(48_000); let data_bytes_size = buf_list.list().buffers[0].data_bytes_size; for i in 0..frame.planes() { - use cap_media_info::PlanarData; - frame.plane_data_mut(i).copy_from_slice( &slice[i * data_bytes_size as usize..(i + 1) * data_bytes_size as usize], ); diff --git a/crates/recording/src/sources/screen_capture/mod.rs b/crates/recording/src/sources/screen_capture/mod.rs index 56794362c8..27075375ea 100644 --- a/crates/recording/src/sources/screen_capture/mod.rs +++ b/crates/recording/src/sources/screen_capture/mod.rs @@ -198,6 +198,8 @@ pub struct ScreenCaptureSource { audio_tx: Option>, start_time: SystemTime, _phantom: std::marker::PhantomData, + #[cfg(windows)] + d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, } impl std::fmt::Debug for ScreenCaptureSource { @@ -236,12 +238,14 @@ impl Clone for ScreenCaptureSource, @@ -251,6 +255,12 @@ struct Config { show_cursor: bool, } +impl Config { + pub fn fps(&self) -> u32 { + self.fps + } +} + #[derive(Debug, Clone, thiserror::Error)] pub enum ScreenCaptureInitError { #[error("NoDisplay")] @@ -271,6 +281,7 @@ impl ScreenCaptureSource { audio_tx: Option>, start_time: SystemTime, tokio_handle: tokio::runtime::Handle, + #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, ) -> Result { cap_fail::fail!("ScreenCaptureSource::init"); @@ -395,9 +406,20 @@ impl ScreenCaptureSource { tokio_handle, start_time, _phantom: std::marker::PhantomData, + #[cfg(windows)] + d3d_device, }) } + #[cfg(windows)] + pub fn d3d_device(&self) -> &::windows::Win32::Graphics::Direct3D11::ID3D11Device { + &self.d3d_device + } + + pub fn config(&self) -> &Config { + &self.config + } + pub fn info(&self) -> VideoInfo { self.video_info } diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 0898d2ae0b..1bed88387c 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -1,5 +1,9 @@ use super::*; -use ::windows::{Graphics::Capture::GraphicsCaptureItem, Win32::Graphics::Direct3D11::D3D11_BOX}; +use ::windows::{ + Foundation::TimeSpan, + Graphics::Capture::GraphicsCaptureItem, + Win32::Graphics::Direct3D11::{D3D11_BOX, ID3D11Device}, +}; use cap_fail::fail_err; use cpal::traits::{DeviceTrait, HostTrait}; use kameo::prelude::*; @@ -15,14 +19,14 @@ const LOG_INTERVAL: Duration = Duration::from_secs(5); const MAX_DROP_RATE_THRESHOLD: f64 = 0.25; #[derive(Debug)] -pub struct AVFrameCapture; +pub struct Direct3DCapture; -impl AVFrameCapture { - const PIXEL_FORMAT: scap_direct3d::PixelFormat = scap_direct3d::PixelFormat::R8G8B8A8Unorm; +impl Direct3DCapture { + pub const PIXEL_FORMAT: scap_direct3d::PixelFormat = scap_direct3d::PixelFormat::R8G8B8A8Unorm; } -impl ScreenCaptureFormat for AVFrameCapture { - type VideoFormat = ffmpeg::frame::Video; +impl ScreenCaptureFormat for Direct3DCapture { + type VideoFormat = scap_direct3d::Frame; fn pixel_format() -> ffmpeg::format::Pixel { scap_direct3d::PixelFormat::R8G8B8A8Unorm.as_ffmpeg() @@ -48,7 +52,7 @@ struct FrameHandler { last_cleanup: Instant, last_log: Instant, frame_events: VecDeque<(Instant, bool)>, - video_tx: Sender<(ffmpeg::frame::Video, f64)>, + video_tx: Sender<(scap_direct3d::Frame, f64)>, } impl Actor for FrameHandler { @@ -120,22 +124,15 @@ impl Message for FrameHandler { async fn handle( &mut self, - mut msg: NewFrame, + msg: NewFrame, ctx: &mut kameo::prelude::Context, ) -> Self::Reply { let Ok(elapsed) = msg.display_time.duration_since(self.start_time) else { return; }; - msg.ff_frame.set_pts(Some( - (elapsed.as_secs_f64() * AV_TIME_BASE_Q.den as f64) as i64, - )); - let now = Instant::now(); - let frame_dropped = match self - .video_tx - .try_send((msg.ff_frame, elapsed.as_secs_f64())) - { + let frame_dropped = match self.video_tx.try_send((msg.frame, elapsed.as_secs_f64())) { Err(flume::TrySendError::Disconnected(_)) => { warn!("Pipeline disconnected"); let _ = ctx.actor_ref().stop_gracefully().await; @@ -204,7 +201,7 @@ enum SourceError { Closed, } -impl PipelineSourceTask for ScreenCaptureSource { +impl PipelineSourceTask for ScreenCaptureSource { // #[instrument(skip_all)] fn run( &mut self, @@ -215,6 +212,7 @@ impl PipelineSourceTask for ScreenCaptureSource { let audio_tx = self.audio_tx.clone(); let start_time = self.start_time; + let d3d_device = self.d3d_device.clone(); // Frame drop rate tracking state let config = self.config.clone(); @@ -222,7 +220,8 @@ impl PipelineSourceTask for ScreenCaptureSource { self.tokio_handle .block_on(async move { let (error_tx, error_rx) = flume::bounded(1); - let capturer = ScreenCaptureActor::spawn(ScreenCaptureActor::new(error_tx)); + let capturer = + ScreenCaptureActor::spawn(ScreenCaptureActor::new(error_tx, d3d_device)); let frame_handler = FrameHandler::spawn(FrameHandler { capturer: capturer.downgrade(), @@ -235,7 +234,7 @@ impl PipelineSourceTask for ScreenCaptureSource { }); let mut settings = scap_direct3d::Settings { - pixel_format: AVFrameCapture::PIXEL_FORMAT, + pixel_format: Direct3DCapture::PIXEL_FORMAT, crop: config.crop_bounds.map(|b| { let position = b.position(); let size = b.size().map(|v| (v / 2.0).floor() * 2.0); @@ -249,6 +248,7 @@ impl PipelineSourceTask for ScreenCaptureSource { back: 1, } }), + min_update_interval: Some(Duration::from_millis(16)), ..Default::default() }; @@ -348,13 +348,15 @@ impl PipelineSourceTask for ScreenCaptureSource { struct ScreenCaptureActor { capture_handle: Option, error_tx: Sender<()>, + d3d_device: ID3D11Device, } impl ScreenCaptureActor { - pub fn new(error_tx: Sender<()>) -> Self { + pub fn new(error_tx: Sender<()>, d3d_device: ID3D11Device) -> Self { Self { capture_handle: None, error_tx, + d3d_device, } } } @@ -374,11 +376,11 @@ pub enum StartCapturingError { #[error("CreateCapturer/{0}")] CreateCapturer(scap_direct3d::NewCapturerError), #[error("StartCapturer/{0}")] - StartCapturer(scap_direct3d::StartCapturerError), + StartCapturer(::windows::core::Error), } pub struct NewFrame { - pub ff_frame: ffmpeg::frame::Video, + pub frame: scap_direct3d::Frame, pub display_time: SystemTime, } @@ -401,32 +403,35 @@ impl Message for ScreenCaptureActor { trace!("Starting capturer with settings: {:?}", &msg.settings); - let mut capture_handle = scap_direct3d::Capturer::new(msg.target, msg.settings) - .map_err(StartCapturingError::CreateCapturer)?; - let error_tx = self.error_tx.clone(); - capture_handle - .start( - move |frame| { - let display_time = SystemTime::now(); - let ff_frame = frame.as_ffmpeg().unwrap(); - - let _ = msg - .frame_handler - .tell(NewFrame { - ff_frame, - display_time, - }) - .try_send(); - - Ok(()) - }, - move || { - let _ = error_tx.send(()); - Ok(()) - }, - ) + let mut capture_handle = scap_direct3d::Capturer::new( + msg.target, + msg.settings, + move |frame| { + let display_time = SystemTime::now(); + + let _ = msg + .frame_handler + .tell(NewFrame { + frame, + display_time, + }) + .try_send(); + + Ok(()) + }, + move || { + let _ = error_tx.send(()); + + Ok(()) + }, + Some(self.d3d_device.clone()), + ) + .map_err(StartCapturingError::CreateCapturer)?; + + capture_handle + .start() .map_err(StartCapturingError::StartCapturer)?; info!("Capturer started"); @@ -489,9 +494,9 @@ pub mod audio { return; }; - ff_frame.set_pts(Some( - (elapsed.as_secs_f64() * AV_TIME_BASE_Q.den as f64) as i64, - )); + let rate = ff_frame.rate(); + + ff_frame.set_pts(Some((elapsed.as_secs_f64() * rate as f64) as i64)); let _ = audio_tx.send((ff_frame, elapsed.as_secs_f64())); i += 1; diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 006dda9eed..4042897095 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -6,7 +6,7 @@ use crate::{ pipeline::Pipeline, sources::{AudioInputSource, CameraSource, ScreenCaptureFormat, ScreenCaptureTarget}, }; -use cap_media_encoders::{H264Encoder, MP4File, OggFile, OpusEncoder}; +use cap_enc_ffmpeg::{H264Encoder, MP4File, OggFile, OpusEncoder}; use cap_media_info::VideoInfo; use cap_project::{CursorEvents, StudioRecordingMeta}; use cap_utils::spawn_actor; @@ -676,14 +676,20 @@ async fn create_segment_pipeline( .cursor_crop() .ok_or(CreateSegmentPipelineError::NoBounds)?; + #[cfg(windows)] + let d3d_device = crate::capture_pipeline::create_d3d_device().unwrap(); + let (screen_source, screen_rx) = create_screen_capture( &capture_target, !custom_cursor_capture, 120, system_audio.0, start_time, + #[cfg(windows)] + d3d_device, ) - .await?; + .await + .unwrap(); let dir = ensure_dir(&segments_dir.join(format!("segment-{index}")))?; @@ -701,7 +707,8 @@ async fn create_segment_pipeline( pipeline_builder, (screen_source, screen_rx), screen_output_path.clone(), - )?; + ) + .unwrap(); pipeline_builder = pipeline_builder_; info!( diff --git a/crates/rendering/src/composite_frame.rs b/crates/rendering/src/composite_frame.rs index e648118436..d57c44b954 100644 --- a/crates/rendering/src/composite_frame.rs +++ b/crates/rendering/src/composite_frame.rs @@ -133,7 +133,7 @@ impl CompositeVideoFramePipeline { address_mode_w: wgpu::AddressMode::ClampToEdge, mag_filter: wgpu::FilterMode::Linear, min_filter: wgpu::FilterMode::Linear, - mipmap_filter: wgpu::FilterMode::Nearest, + mipmap_filter: wgpu::FilterMode::Linear, ..Default::default() }), ); diff --git a/crates/rendering/src/shaders/composite-video-frame.wgsl b/crates/rendering/src/shaders/composite-video-frame.wgsl index 4f47f4fb1d..a0ddd0bb3f 100644 --- a/crates/rendering/src/shaders/composite-video-frame.wgsl +++ b/crates/rendering/src/shaders/composite-video-frame.wgsl @@ -160,7 +160,43 @@ fn sample_texture(uv: vec2, crop_bounds_uv: vec4) -> vec4 { } let cropped_uv = sample_uv * (crop_bounds_uv.zw - crop_bounds_uv.xy) + crop_bounds_uv.xy; - return vec4(textureSample(frame_texture, frame_sampler, cropped_uv).rgb, 1.0); + + // Calculate downscaling ratio + let source_size = uniforms.frame_size * (crop_bounds_uv.zw - crop_bounds_uv.xy); + let target_size = uniforms.target_size; + let scale_ratio = source_size / target_size; + let is_downscaling = max(scale_ratio.x, scale_ratio.y) > 1.1; + + // Sample the center pixel + let center_color = textureSample(frame_texture, frame_sampler, cropped_uv).rgb; + + // Apply sharpening when downscaling to preserve text clarity + if is_downscaling { + let texel_size = 1.0 / uniforms.frame_size; + + // Sample neighboring pixels for unsharp mask + let offset_x = vec2(texel_size.x, 0.0); + let offset_y = vec2(0.0, texel_size.y); + + // 4-tap sampling for edge detection + let left = textureSample(frame_texture, frame_sampler, cropped_uv - offset_x).rgb; + let right = textureSample(frame_texture, frame_sampler, cropped_uv + offset_x).rgb; + let top = textureSample(frame_texture, frame_sampler, cropped_uv - offset_y).rgb; + let bottom = textureSample(frame_texture, frame_sampler, cropped_uv + offset_y).rgb; + + // Calculate the blurred version (average of neighbors) + let blurred = (left + right + top + bottom) * 0.25; + + // Unsharp mask: enhance the difference between center and blur + // Strength is adaptive based on downscale ratio + let sharpness = min(scale_ratio.x * 0.3, 0.7); // Cap at 0.7 to avoid over-sharpening + let sharpened = center_color + (center_color - blurred) * sharpness; + + // Clamp to avoid color artifacts + return vec4(clamp(sharpened, vec3(0.0), vec3(1.0)), 1.0); + } + + return vec4(center_color, 1.0); } return vec4(0.0); diff --git a/crates/scap-cpal/src/lib.rs b/crates/scap-cpal/src/lib.rs index e4450b2ffd..1c8507991b 100644 --- a/crates/scap-cpal/src/lib.rs +++ b/crates/scap-cpal/src/lib.rs @@ -1,6 +1,6 @@ use cpal::{ - BuildStreamError, DefaultStreamConfigError, InputCallbackInfo, PauseStreamError, - PlayStreamError, Stream, StreamConfig, StreamError, traits::StreamTrait, + InputCallbackInfo, PauseStreamError, PlayStreamError, Stream, StreamConfig, StreamError, + traits::StreamTrait, }; use thiserror::Error; diff --git a/crates/scap-direct3d/examples/cli.rs b/crates/scap-direct3d/examples/cli.rs index 30be599c9f..cc8020267b 100644 --- a/crates/scap-direct3d/examples/cli.rs +++ b/crates/scap-direct3d/examples/cli.rs @@ -5,1124 +5,53 @@ fn main() { #[cfg(windows)] mod windows { - use ::windows::Graphics::SizeInt32; - use ::windows::Storage::FileAccessMode; - use ::windows::Win32::Media::MediaFoundation::{MFSTARTUP_FULL, MFStartup}; - use ::windows::Win32::System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize}; - use ::windows::Win32::UI::WindowsAndMessaging::{ - DispatchMessageW, GetMessageW, MSG, WM_HOTKEY, - }; - use ::windows::{ - Storage::{CreationCollisionOption, StorageFolder}, - Win32::{Foundation::MAX_PATH, Storage::FileSystem::GetFullPathNameW}, - core::HSTRING, - }; - use cap_displays::*; use scap_direct3d::{Capturer, PixelFormat, Settings}; - use std::time::Instant; - use std::{path::Path, sync::Arc, time::Duration}; - - use super::*; + use scap_ffmpeg::*; + use scap_targets::*; + use std::time::Duration; + use windows::Win32::Graphics::Direct3D11::D3D11_BOX; pub fn main() { - unsafe { - RoInitialize(RO_INIT_MULTITHREADED).unwrap(); - } - unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL).unwrap() } - let display = Display::primary(); let display = display.raw_handle(); - let (frame_tx, frame_rx) = std::sync::mpsc::channel(); - - let mut capturer = Capturer::new( - display.try_as_capture_item().unwrap(), - Settings { - is_border_required: Some(true), - is_cursor_capture_enabled: Some(true), - pixel_format: PixelFormat::R8G8B8A8Unorm, - // crop: Some(D3D11_BOX { - // left: 0, - // top: 0, - // right: 500, - // bottom: 400, - // front: 0, - // back: 1, - // }), - ..Default::default() - }, - ) - .unwrap(); - - let mut encoder_devices = VideoEncoderDevice::enumerate().unwrap(); - let encoder_device = encoder_devices.swap_remove(0); - - let mut video_encoder = VideoEncoder::new( - &encoder_device, - capturer.d3d_device().clone(), - SizeInt32 { - Width: 3340, - Height: 1440, - }, - SizeInt32 { - Width: 3340, - Height: 1440, - }, - 12_000_000, - 60, - ) - .unwrap(); - let output_type = video_encoder.output_type().clone(); - - // Create our file - let path = unsafe { - let mut new_path = vec![0u16; MAX_PATH as usize]; - let length = - GetFullPathNameW(&HSTRING::from("recording.mp4"), Some(&mut new_path), None); - new_path.resize(length as usize, 0); - String::from_utf16(&new_path).unwrap() - }; - let path = Path::new(&path); - let parent_folder_path = path.parent().unwrap(); - let parent_folder = StorageFolder::GetFolderFromPathAsync(&HSTRING::from( - parent_folder_path.as_os_str().to_str().unwrap(), - )) - .unwrap() - .get() - .unwrap(); - let file_name = path.file_name().unwrap(); - let file = parent_folder - .CreateFileAsync( - &HSTRING::from(file_name.to_str().unwrap()), - CreationCollisionOption::ReplaceExisting, - ) - .unwrap() - .get() - .unwrap(); - - let stream = file - .OpenAsync(FileAccessMode::ReadWrite) - .unwrap() - .get() - .unwrap(); - - video_encoder.set_sample_requested_callback(move || Ok(frame_rx.recv().ok())); - - let sample_writer = Arc::new(SampleWriter::new(stream, &output_type).unwrap()); - video_encoder.set_sample_rendered_callback({ - let sample_writer = sample_writer.clone(); - move |sample| { - dbg!(sample.sample()); - sample_writer.write(sample.sample()) - } - }); - - sample_writer.start().unwrap(); - - let mut first_timestamp = None; - - capturer - .start( - move |frame| { - let frame_time = frame.inner().SystemRelativeTime().unwrap(); - - let first_timestamp = first_timestamp.get_or_insert(frame_time); - - let _ = frame_tx.send(VideoEncoderInputSample::new( - ::windows::Foundation::TimeSpan { - Duration: frame_time.Duration - first_timestamp.Duration, - }, - frame.texture().clone(), - )); - // dbg!(&frame); - - // let ff_frame = frame.as_ffmpeg()?; - // dbg!(ff_frame.width(), ff_frame.height(), ff_frame.format()); - - Ok(()) - }, - || Ok(()), - ) - .unwrap(); - - video_encoder.try_start().unwrap(); - - std::thread::sleep(Duration::from_secs(10)); - - video_encoder.stop().unwrap(); - sample_writer.stop().unwrap(); - capturer.stop().unwrap(); - - // std::thread::sleep(Duration::from_secs(3)); - } - - fn pump_messages() -> ::windows::core::Result<()> { - // let _hot_key = HotKey::new(MOD_SHIFT | MOD_CONTROL, 0x52 /* R */)?; - // println!("Press SHIFT+CTRL+R to start/stop the recording..."); - let start = Instant::now(); - unsafe { - let mut message = MSG::default(); - while GetMessageW(&mut message, None, 0, 0).into() { - dbg!(message.message); - if start.elapsed().as_secs_f64() > 3.0 { - break; - } - DispatchMessageW(&message); - } - } - Ok(()) - } -} - -use encoder::*; -mod encoder { - use std::{ - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - thread::JoinHandle, - }; - - use ::windows::{ - Foundation::TimeSpan, - Graphics::SizeInt32, - Win32::{ - Foundation::E_NOTIMPL, - Graphics::Direct3D11::{ID3D11Device, ID3D11Texture2D}, - Media::MediaFoundation::{ - IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, - IMFSample, IMFTransform, MEDIA_EVENT_GENERATOR_GET_EVENT_FLAGS, - METransformHaveOutput, METransformNeedInput, MF_E_INVALIDMEDIATYPE, - MF_E_NO_MORE_TYPES, MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_TYPE, - MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, - MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, - MF_MT_SUBTYPE, MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, - MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType, - MFCreateSample, MFMediaType_Video, MFSTARTUP_FULL, MFStartup, - MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, - MFT_MESSAGE_NOTIFY_END_OF_STREAM, MFT_MESSAGE_NOTIFY_END_STREAMING, - MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, - MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY, MFVideoFormat_H264, - MFVideoFormat_NV12, MFVideoInterlace_Progressive, - }, - }, - core::{Error, Interface, Result}, - }; - - use super::*; - - // use crate::media::{MF_VERSION, MFSetAttributeRatio, MFSetAttributeSize}; - - // use super::encoder_device::VideoEncoderDevice; - - pub struct VideoEncoderInputSample { - timestamp: TimeSpan, - texture: ID3D11Texture2D, - } - - impl VideoEncoderInputSample { - pub fn new(timestamp: TimeSpan, texture: ID3D11Texture2D) -> Self { - Self { timestamp, texture } - } - } - - pub struct VideoEncoderOutputSample { - sample: IMFSample, - } - - impl VideoEncoderOutputSample { - pub fn sample(&self) -> &IMFSample { - &self.sample - } - } - - pub struct VideoEncoder { - inner: Option, - output_type: IMFMediaType, - started: AtomicBool, - should_stop: Arc, - encoder_thread_handle: Option>>, - } - - struct VideoEncoderInner { - _d3d_device: ID3D11Device, - _media_device_manager: IMFDXGIDeviceManager, - _device_manager_reset_token: u32, - - transform: IMFTransform, - event_generator: IMFMediaEventGenerator, - input_stream_id: u32, - output_stream_id: u32, - - sample_requested_callback: - Option Result>>>, - sample_rendered_callback: - Option Result<()>>>, - - should_stop: Arc, - } - - impl VideoEncoder { - pub fn new( - encoder_device: &VideoEncoderDevice, - d3d_device: ID3D11Device, - input_resolution: SizeInt32, - output_resolution: SizeInt32, - bit_rate: u32, - frame_rate: u32, - ) -> Result { - let transform = encoder_device.create_transform()?; - - // Create MF device manager - let mut device_manager_reset_token: u32 = 0; - let media_device_manager = { - let mut media_device_manager = None; - unsafe { - MFCreateDXGIDeviceManager( - &mut device_manager_reset_token, - &mut media_device_manager, - )? - }; - media_device_manager.unwrap() - }; - unsafe { media_device_manager.ResetDevice(&d3d_device, device_manager_reset_token)? }; - - // Setup MFTransform - let event_generator: IMFMediaEventGenerator = transform.cast()?; - let attributes = unsafe { transform.GetAttributes()? }; - unsafe { - attributes.SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1)?; - attributes.SetUINT32(&MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1)?; - }; - - let mut number_of_input_streams = 0; - let mut number_of_output_streams = 0; - unsafe { - transform - .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams)? - }; - let (input_stream_ids, output_stream_ids) = { - let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; - let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; - let result = unsafe { - transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) - }; - match result { - Ok(_) => {} - Err(error) => { - // https://docs.microsoft.com/en-us/windows/win32/api/mftransform/nf-mftransform-imftransform-getstreamids - // This method can return E_NOTIMPL if both of the following conditions are true: - // * The transform has a fixed number of streams. - // * The streams are numbered consecutively from 0 to n – 1, where n is the - // number of input streams or output streams. In other words, the first - // input stream is 0, the second is 1, and so on; and the first output - // stream is 0, the second is 1, and so on. - if error.code() == E_NOTIMPL { - for i in 0..number_of_input_streams { - input_stream_ids[i as usize] = i; - } - for i in 0..number_of_output_streams { - output_stream_ids[i as usize] = i; - } - } else { - return Err(error); - } - } - } - (input_stream_ids, output_stream_ids) - }; - let input_stream_id = input_stream_ids[0]; - let output_stream_id = output_stream_ids[0]; - - // TOOD: Avoid this AddRef? - unsafe { - let temp = media_device_manager.clone(); - transform.ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, std::mem::transmute(temp))?; - }; - - let output_type = unsafe { - let output_type = MFCreateMediaType()?; - let attributes: IMFAttributes = output_type.cast()?; - output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; - output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; - output_type.SetUINT32(&MF_MT_AVG_BITRATE, bit_rate)?; - MFSetAttributeSize( - &attributes, - &MF_MT_FRAME_SIZE, - output_resolution.Width as u32, - output_resolution.Height as u32, - )?; - MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; - MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; - output_type - .SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; - output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; - transform.SetOutputType(output_stream_id, &output_type, 0)?; - output_type - }; - let input_type: Option = unsafe { - let mut count = 0; - loop { - let result = transform.GetInputAvailableType(input_stream_id, count); - if let Err(error) = &result { - if error.code() == MF_E_NO_MORE_TYPES { - break None; - } - } - - let input_type = result?; - let attributes: IMFAttributes = input_type.cast()?; - input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; - input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; - MFSetAttributeSize( - &attributes, - &MF_MT_FRAME_SIZE, - input_resolution.Width as u32, - input_resolution.Height as u32, - )?; - MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, 60, 1)?; - let result = transform.SetInputType( - input_stream_id, - &input_type, - MFT_SET_TYPE_TEST_ONLY.0 as u32, - ); - if let Err(error) = &result { - if error.code() == MF_E_INVALIDMEDIATYPE { - count += 1; - continue; - } - } - result?; - break Some(input_type); - } - }; - if let Some(input_type) = input_type { - unsafe { transform.SetInputType(input_stream_id, &input_type, 0)? }; - } else { - return Err(Error::new( - MF_E_TRANSFORM_TYPE_NOT_SET, - "No suitable input type found! Try a different set of encoding settings.", - )); - } - - let should_stop = Arc::new(AtomicBool::new(false)); - let inner = VideoEncoderInner { - _d3d_device: d3d_device, - _media_device_manager: media_device_manager, - _device_manager_reset_token: device_manager_reset_token, - - transform, - event_generator, - input_stream_id, - output_stream_id, - - sample_requested_callback: None, - sample_rendered_callback: None, - - should_stop: should_stop.clone(), - }; - - Ok(Self { - inner: Some(inner), - output_type, - started: AtomicBool::new(false), - should_stop, - encoder_thread_handle: None, - }) - } - - pub fn try_start(&mut self) -> Result { - let mut result = false; - if self - .started - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - { - let mut inner = self.inner.take().unwrap(); - - // Callbacks must both be set - if inner.sample_rendered_callback.is_none() - || inner.sample_requested_callback.is_none() - { - panic!("Sample requested and rendered callbacks must be set before starting"); - } - - // Start a seperate thread to drive the transform - self.encoder_thread_handle = Some(std::thread::spawn(move || -> Result<()> { - unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL)? } - let result = inner.encode(); - if result.is_err() { - println!("Recording stopped unexpectedly!"); - } - result - })); - result = true; - } - Ok(result) - } - - pub fn stop(&mut self) -> Result<()> { - if self.started.load(Ordering::SeqCst) { - assert!( - self.should_stop - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - ); - self.wait_for_completion()?; - } - Ok(()) - } - - fn wait_for_completion(&mut self) -> Result<()> { - let handle = self.encoder_thread_handle.take().unwrap(); - handle.join().unwrap() - } - - pub fn set_sample_requested_callback< - F: 'static + Send + FnMut() -> Result>, - >( - &mut self, - callback: F, - ) { - self.inner.as_mut().unwrap().sample_requested_callback = Some(Box::new(callback)); - } - - pub fn set_sample_rendered_callback< - F: 'static + Send + FnMut(VideoEncoderOutputSample) -> Result<()>, - >( - &mut self, - callback: F, - ) { - self.inner.as_mut().unwrap().sample_rendered_callback = Some(Box::new(callback)); - } - - pub fn output_type(&self) -> &IMFMediaType { - &self.output_type - } - } - - unsafe impl Send for VideoEncoderInner {} - // Workaround for: - // warning: constant in pattern `METransformNeedInput` should have an upper case name - // --> src\video\encoder.rs:XXX:YY - // | - // XXX | METransformNeedInput => { - // | ^^^^^^^^^^^^^^^^^^^^ help: convert the identifier to upper case: `METRANSFORM_NEED_INPUT` - // | - // = note: `#[warn(non_upper_case_globals)]` on by default - const MEDIA_ENGINE_TRANFORM_NEED_INPUT: MF_EVENT_TYPE = METransformNeedInput; - const MEDIA_ENGINE_TRANFORM_HAVE_OUTPUT: MF_EVENT_TYPE = METransformHaveOutput; - impl VideoEncoderInner { - fn encode(&mut self) -> Result<()> { - unsafe { - self.transform - .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; - self.transform - .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0)?; - self.transform - .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)?; - - let mut should_exit = false; - while !should_exit { - let event = self - .event_generator - .GetEvent(MEDIA_EVENT_GENERATOR_GET_EVENT_FLAGS(0))?; - - let event_type = MF_EVENT_TYPE(event.GetType()? as i32); - match event_type { - MEDIA_ENGINE_TRANFORM_NEED_INPUT => { - should_exit = self.on_transform_input_requested()?; - } - MEDIA_ENGINE_TRANFORM_HAVE_OUTPUT => { - self.on_transform_output_ready()?; - } - _ => { - panic!("Unknown media event type: {}", event_type.0); - } - } - } - - self.transform - .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; - self.transform - .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0)?; - self.transform - .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; - } - Ok(()) - } - - fn on_transform_input_requested(&mut self) -> Result { - let mut should_exit = true; - if !self.should_stop.load(Ordering::SeqCst) { - if let Some(sample) = self.sample_requested_callback.as_mut().unwrap()()? { - let input_buffer = unsafe { - MFCreateDXGISurfaceBuffer(&ID3D11Texture2D::IID, &sample.texture, 0, false)? - }; - let mf_sample = unsafe { MFCreateSample()? }; - unsafe { - mf_sample.AddBuffer(&input_buffer)?; - mf_sample.SetSampleTime(sample.timestamp.Duration)?; - self.transform - .ProcessInput(self.input_stream_id, &mf_sample, 0)?; - }; - should_exit = false; - } - } - Ok(should_exit) - } - - fn on_transform_output_ready(&mut self) -> Result<()> { - let mut status = 0; - let output_buffer = MFT_OUTPUT_DATA_BUFFER { - dwStreamID: self.output_stream_id, - ..Default::default() - }; - - let sample = unsafe { - let mut output_buffers = [output_buffer]; - self.transform - .ProcessOutput(0, &mut output_buffers, &mut status)?; - output_buffers[0].pSample.as_ref().unwrap().clone() - }; - - let output_sample = VideoEncoderOutputSample { sample }; - self.sample_rendered_callback.as_mut().unwrap()(output_sample)?; - Ok(()) - } - } -} - -use encoder_device::*; -mod encoder_device { - use ::windows::{ - Win32::Media::MediaFoundation::{ - IMFActivate, IMFTransform, MFMediaType_Video, MFT_CATEGORY_VIDEO_ENCODER, - MFT_ENUM_FLAG_HARDWARE, MFT_ENUM_FLAG_SORTANDFILTER, MFT_ENUM_FLAG_TRANSCODE_ONLY, - MFT_FRIENDLY_NAME_Attribute, MFT_REGISTER_TYPE_INFO, MFVideoFormat_H264, - }, - core::{Interface, Result}, - }; - - use super::*; - - #[derive(Clone)] - pub struct VideoEncoderDevice { - source: IMFActivate, - display_name: String, - } - - impl VideoEncoderDevice { - pub fn enumerate() -> Result> { - let output_info = MFT_REGISTER_TYPE_INFO { - guidMajorType: MFMediaType_Video, - guidSubtype: MFVideoFormat_H264, - }; - let encoders = enumerate_mfts( - &MFT_CATEGORY_VIDEO_ENCODER, - MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY | MFT_ENUM_FLAG_SORTANDFILTER, - None, - Some(&output_info), - )?; - let mut encoder_devices = Vec::new(); - for encoder in encoders { - let display_name = if let Some(display_name) = - get_string_attribute(&encoder.cast()?, &MFT_FRIENDLY_NAME_Attribute)? - { - display_name - } else { - "Unknown".to_owned() - }; - let encoder_device = VideoEncoderDevice { - source: encoder, - display_name, - }; - encoder_devices.push(encoder_device); - } - Ok(encoder_devices) - } - - pub fn display_name(&self) -> &str { - &self.display_name - } - - pub fn create_transform(&self) -> Result { - unsafe { self.source.ActivateObject() } - } - } -} - -use media::*; -mod media { - use ::windows::{ - Win32::Media::MediaFoundation::{ - IMFActivate, IMFAttributes, MF_E_ATTRIBUTENOTFOUND, MFT_ENUM_FLAG, - MFT_REGISTER_TYPE_INFO, MFTEnumEx, - }, - core::{Array, GUID, Result}, - }; - - use super::*; - - pub fn enumerate_mfts( - category: &GUID, - flags: MFT_ENUM_FLAG, - input_type: Option<&MFT_REGISTER_TYPE_INFO>, - output_type: Option<&MFT_REGISTER_TYPE_INFO>, - ) -> Result> { - let mut transform_sources = Vec::new(); - let mfactivate_list = unsafe { - let mut data = std::ptr::null_mut(); - let mut len = 0; - MFTEnumEx( - *category, - flags, - input_type.map(|info| info as *const _), - output_type.map(|info| info as *const _), - &mut data, - &mut len, - )?; - Array::::from_raw_parts(data as _, len) - }; - if !mfactivate_list.is_empty() { - for mfactivate in mfactivate_list.as_slice() { - let transform_source = mfactivate.clone().unwrap(); - transform_sources.push(transform_source); - } - } - Ok(transform_sources) - } - - pub fn get_string_attribute( - attributes: &IMFAttributes, - attribute_guid: &GUID, - ) -> Result> { - unsafe { - match attributes.GetStringLength(attribute_guid) { - Ok(mut length) => { - let mut result = vec![0u16; (length + 1) as usize]; - attributes.GetString(attribute_guid, &mut result, Some(&mut length))?; - result.resize(length as usize, 0); - Ok(Some(String::from_utf16(&result).unwrap())) - } - Err(error) => { - if error.code() == MF_E_ATTRIBUTENOTFOUND { - Ok(None) - } else { - Err(error) - } - } - } - } - } - - // These inlined helpers aren't represented in the metadata - - // This is the value for Win7+ - pub const MF_VERSION: u32 = 131184; - - fn pack_2_u32_as_u64(high: u32, low: u32) -> u64 { - ((high as u64) << 32) | low as u64 - } - - #[allow(non_snake_case)] - unsafe fn MFSetAttribute2UINT32asUINT64( - attributes: &IMFAttributes, - key: &GUID, - high: u32, - low: u32, - ) -> Result<()> { - unsafe { attributes.SetUINT64(key, pack_2_u32_as_u64(high, low)) } - } - - #[allow(non_snake_case)] - pub unsafe fn MFSetAttributeSize( - attributes: &IMFAttributes, - key: &GUID, - width: u32, - height: u32, - ) -> Result<()> { - unsafe { MFSetAttribute2UINT32asUINT64(attributes, key, width, height) } - } - - #[allow(non_snake_case)] - pub unsafe fn MFSetAttributeRatio( - attributes: &IMFAttributes, - key: &GUID, - numerator: u32, - denominator: u32, - ) -> Result<()> { - unsafe { MFSetAttribute2UINT32asUINT64(attributes, key, numerator, denominator) } - } -} - -use encoding_session::*; -mod encoding_session { - - use std::sync::Arc; - - use ::windows::{ - Foundation::TimeSpan, - Graphics::{ - Capture::{Direct3D11CaptureFrame, GraphicsCaptureItem, GraphicsCaptureSession}, - SizeInt32, - }, - Storage::Streams::IRandomAccessStream, - Win32::{ - Graphics::{ - Direct3D11::{ - D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, - D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, ID3D11Device, ID3D11DeviceContext, - ID3D11RenderTargetView, ID3D11Texture2D, - }, - Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}, - }, - Media::MediaFoundation::{ - IMFMediaType, IMFSample, IMFSinkWriter, MFCreateAttributes, - MFCreateMFByteStreamOnStreamEx, MFCreateSinkWriterFromURL, - }, - System::WinRT::Direct3D11::IDirect3DDxgiInterfaceAccess, - }, - core::{HSTRING, Interface, Result}, - }; - - use super::*; - - // capture::CaptureFrameGenerator, - // d3d::get_d3d_interface_from_object, - // video::{ - // CLEAR_COLOR, - // encoding_session::{VideoEncoderSessionFactory, VideoEncodingSession}, - // util::ensure_even_size, - // }, - // }; - - // use super::{ - // encoder::{VideoEncoder, VideoEncoderInputSample}, - // encoder_device::VideoEncoderDevice, - // processor::VideoProcessor, - // }; - - pub struct SampleWriter { - _stream: IRandomAccessStream, - sink_writer: IMFSinkWriter, - sink_writer_stream_index: u32, - } - - unsafe impl Send for SampleWriter {} - unsafe impl Sync for SampleWriter {} - impl SampleWriter { - pub fn new(stream: IRandomAccessStream, output_type: &IMFMediaType) -> Result { - let empty_attributes = unsafe { - let mut attributes = None; - MFCreateAttributes(&mut attributes, 0)?; - attributes.unwrap() - }; - let sink_writer = unsafe { - let byte_stream = MFCreateMFByteStreamOnStreamEx(&stream)?; - MFCreateSinkWriterFromURL(&HSTRING::from(".mp4"), &byte_stream, &empty_attributes)? - }; - let sink_writer_stream_index = unsafe { sink_writer.AddStream(output_type)? }; - unsafe { - sink_writer.SetInputMediaType( - sink_writer_stream_index, - output_type, - &empty_attributes, - )? - }; - - Ok(Self { - _stream: stream, - sink_writer, - sink_writer_stream_index, - }) - } - - pub fn start(&self) -> Result<()> { - unsafe { self.sink_writer.BeginWriting() } - } - - pub fn stop(&self) -> Result<()> { - unsafe { self.sink_writer.Finalize() } - } - - pub fn write(&self, sample: &IMFSample) -> Result<()> { - unsafe { - self.sink_writer - .WriteSample(self.sink_writer_stream_index, sample) - } - } - } - - pub fn get_d3d_interface_from_object(object: &S) -> Result { - let access: IDirect3DDxgiInterfaceAccess = object.cast()?; - let object = unsafe { access.GetInterface::()? }; - Ok(object) - } -} - -use processor::*; -mod processor { - use ::windows::{ - Graphics::{RectInt32, SizeInt32}, - Win32::{ - Foundation::RECT, - Graphics::{ - Direct3D11::{ - D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BIND_VIDEO_ENCODER, - D3D11_TEX2D_VPIV, D3D11_TEX2D_VPOV, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, - D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, D3D11_VIDEO_PROCESSOR_COLOR_SPACE, - D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, - D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0, - D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255, - D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235, - D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, - D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0, D3D11_VIDEO_PROCESSOR_STREAM, - D3D11_VIDEO_USAGE_OPTIMAL_QUALITY, D3D11_VPIV_DIMENSION_TEXTURE2D, - D3D11_VPOV_DIMENSION_TEXTURE2D, ID3D11Device, ID3D11DeviceContext, - ID3D11Texture2D, ID3D11VideoContext, ID3D11VideoDevice, ID3D11VideoProcessor, - ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, - }, - Dxgi::Common::{DXGI_FORMAT, DXGI_RATIONAL, DXGI_SAMPLE_DESC}, - }, - }, - core::{Interface, Result}, - }; - use windows_numerics::Vector2; - - pub struct VideoProcessor { - _d3d_device: ID3D11Device, - d3d_context: ID3D11DeviceContext, - - _video_device: ID3D11VideoDevice, - video_context: ID3D11VideoContext, - video_processor: ID3D11VideoProcessor, - video_output_texture: ID3D11Texture2D, - video_output: ID3D11VideoProcessorOutputView, - video_input_texture: ID3D11Texture2D, - video_input: ID3D11VideoProcessorInputView, - } - - impl VideoProcessor { - pub fn new( - d3d_device: ID3D11Device, - input_format: DXGI_FORMAT, - input_size: SizeInt32, - output_format: DXGI_FORMAT, - output_size: SizeInt32, - ) -> Result { - let d3d_context = unsafe { d3d_device.GetImmediateContext()? }; - - // Setup video conversion - let video_device: ID3D11VideoDevice = d3d_device.cast()?; - let video_context: ID3D11VideoContext = d3d_context.cast()?; - - let video_desc = D3D11_VIDEO_PROCESSOR_CONTENT_DESC { - InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, - InputFrameRate: DXGI_RATIONAL { - Numerator: 60, - Denominator: 1, - }, - InputWidth: input_size.Width as u32, - InputHeight: input_size.Height as u32, - OutputFrameRate: DXGI_RATIONAL { - Numerator: 60, - Denominator: 1, - }, - OutputWidth: input_size.Width as u32, - OutputHeight: input_size.Height as u32, - Usage: D3D11_VIDEO_USAGE_OPTIMAL_QUALITY, - }; - let video_enum = unsafe { video_device.CreateVideoProcessorEnumerator(&video_desc)? }; - - let video_processor = unsafe { video_device.CreateVideoProcessor(&video_enum, 0)? }; - - let mut color_space = D3D11_VIDEO_PROCESSOR_COLOR_SPACE { - _bitfield: 1 | (D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235.0 as u32) << 4, // Usage: 1 (Video processing), Nominal_Range: D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235 - }; - unsafe { - video_context.VideoProcessorSetOutputColorSpace(&video_processor, &color_space) - }; - color_space._bitfield = 1 | (D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255.0 as u32) << 4; // Usage: 1 (Video processing), Nominal_Range: D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255 - unsafe { - video_context.VideoProcessorSetStreamColorSpace(&video_processor, 0, &color_space) - }; - - // If the input and output resolutions don't match, setup the - // video processor to preserve the aspect ratio when scaling. - if input_size.Width != output_size.Width || input_size.Height != output_size.Height { - let dest_rect = compute_dest_rect(&output_size, &input_size); - let rect = RECT { - left: dest_rect.X, - top: dest_rect.Y, - right: dest_rect.X + dest_rect.Width, - bottom: dest_rect.Y + dest_rect.Height, - }; - unsafe { - video_context.VideoProcessorSetStreamDestRect( - &video_processor, - 0, - true, - Some(&rect), - ) - }; - } - - let mut texture_desc = D3D11_TEXTURE2D_DESC { - Width: output_size.Width as u32, - Height: output_size.Height as u32, - ArraySize: 1, - MipLevels: 1, - Format: output_format, - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - ..Default::default() - }, - Usage: D3D11_USAGE_DEFAULT, - BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_VIDEO_ENCODER.0) as u32, - ..Default::default() - }; - let video_output_texture = unsafe { - let mut texture = None; - d3d_device.CreateTexture2D(&texture_desc, None, Some(&mut texture))?; - texture.unwrap() - }; - - let output_view_desc = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC { - ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D, - Anonymous: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 { - Texture2D: D3D11_TEX2D_VPOV { MipSlice: 0 }, - }, - }; - let video_output = unsafe { - let mut output = None; - video_device.CreateVideoProcessorOutputView( - &video_output_texture, - &video_enum, - &output_view_desc, - Some(&mut output), - )?; - output.unwrap() - }; - - texture_desc.Width = input_size.Width as u32; - texture_desc.Height = input_size.Height as u32; - texture_desc.Format = input_format; - texture_desc.BindFlags = - (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32; - let video_input_texture = unsafe { - let mut texture = None; - d3d_device.CreateTexture2D(&texture_desc, None, Some(&mut texture))?; - texture.unwrap() - }; - - let input_view_desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { - ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D, - Anonymous: D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 { - Texture2D: D3D11_TEX2D_VPIV { - MipSlice: 0, - ..Default::default() - }, - }, - ..Default::default() - }; - let video_input = unsafe { - let mut input = None; - video_device.CreateVideoProcessorInputView( - &video_input_texture, - &video_enum, - &input_view_desc, - Some(&mut input), - )?; - input.unwrap() - }; - - Ok(Self { - _d3d_device: d3d_device, - d3d_context, - - _video_device: video_device, - video_context, - video_processor, - video_output_texture, - video_output, - video_input_texture, - video_input, - }) - } - - pub fn output_texture(&self) -> &ID3D11Texture2D { - &self.video_output_texture - } - - pub fn process_texture(&mut self, input_texture: &ID3D11Texture2D) -> Result<()> { - // The caller is responsible for making sure they give us a - // texture that matches the input size we were initialized with. - - unsafe { - // Copy the texture to the video input texture - self.d3d_context - .CopyResource(&self.video_input_texture, input_texture); - - // Convert to NV12 - let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { - Enable: true.into(), - OutputIndex: 0, - InputFrameOrField: 0, - pInputSurface: std::mem::transmute_copy(&self.video_input), - ..Default::default() - }; - self.video_context.VideoProcessorBlt( - &self.video_processor, - &self.video_output, - 0, - &[video_stream], - ) - } - } - } - - fn compute_scale_factor(output_size: Vector2, input_size: Vector2) -> f32 { - let output_ratio = output_size.X / output_size.Y; - let input_ratio = input_size.X / input_size.Y; - - let mut scale_factor = output_size.X / input_size.X; - if output_ratio > input_ratio { - scale_factor = output_size.Y / input_size.Y; - } - - scale_factor - } - - fn compute_dest_rect(output_size: &SizeInt32, input_size: &SizeInt32) -> RectInt32 { - let scale = compute_scale_factor( - Vector2 { - X: output_size.Width as f32, - Y: output_size.Height as f32, - }, - Vector2 { - X: input_size.Width as f32, - Y: input_size.Height as f32, - }, - ); - let new_size = SizeInt32 { - Width: (input_size.Width as f32 * scale) as i32, - Height: (input_size.Height as f32 * scale) as i32, - }; - let mut offset_x = 0; - let mut offset_y = 0; - if new_size.Width != output_size.Width { - offset_x = (output_size.Width - new_size.Width) / 2; - } - if new_size.Height != output_size.Height { - offset_y = (output_size.Height - new_size.Height) / 2; - } - RectInt32 { - X: offset_x, - Y: offset_y, - Width: new_size.Width, - Height: new_size.Height, - } + // let mut capturer = Capturer::new( + // display.try_as_capture_item().unwrap(), + // Settings { + // is_border_required: Some(false), + // is_cursor_capture_enabled: Some(true), + // pixel_format: PixelFormat::R8G8B8A8Unorm, + // // crop: Some(D3D11_BOX { + // // left: 0, + // // top: 0, + // // right: 500, + // // bottom: 400, + // // front: 0, + // // back: 1, + // // }), + // ..Default::default() + // }, + // ) + // .unwrap(); + + // capturer + // .start( + // |frame| { + // dbg!(&frame); + + // let ff_frame = frame.as_ffmpeg()?; + // dbg!(ff_frame.width(), ff_frame.height(), ff_frame.format()); + + // Ok(()) + // }, + // || Ok(()), + // ) + // .unwrap(); + + std::thread::sleep(Duration::from_secs(3)); + + // capturer.stop().unwrap(); + + std::thread::sleep(Duration::from_secs(3)); } } diff --git a/crates/scap-direct3d/examples/mf_encoder.rs b/crates/scap-direct3d/examples/mf_encoder.rs deleted file mode 100644 index 6054dbe08b..0000000000 --- a/crates/scap-direct3d/examples/mf_encoder.rs +++ /dev/null @@ -1,57 +0,0 @@ -fn main() { - #[cfg(windows)] - windows::main(); -} - -#[cfg(windows)] -mod windows { - use cap_displays::*; - use scap_direct3d::{Capturer, PixelFormat, Settings}; - use scap_ffmpeg::*; - use std::time::Duration; - use windows::Win32::Graphics::Direct3D11::D3D11_BOX; - - pub fn main() { - let display = Display::primary(); - let display = display.raw_handle(); - - let mut capturer = Capturer::new( - display.try_as_capture_item().unwrap(), - Settings { - is_border_required: Some(false), - is_cursor_capture_enabled: Some(true), - pixel_format: PixelFormat::R8G8B8A8Unorm, - // crop: Some(D3D11_BOX { - // left: 0, - // top: 0, - // right: 500, - // bottom: 400, - // front: 0, - // back: 1, - // }), - ..Default::default() - }, - ) - .unwrap(); - - capturer - .start( - |frame| { - dbg!(&frame); - - let ff_frame = frame.as_ffmpeg()?; - dbg!(ff_frame.width(), ff_frame.height(), ff_frame.format()); - - Ok(()) - }, - || Ok(()), - ) - .unwrap(); - - std::thread::sleep(Duration::from_secs(3)); - - capturer.stop().unwrap(); - - std::thread::sleep(Duration::from_secs(3)); - } -} diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index d62f5db4c7..73edb47a6c 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -34,7 +34,10 @@ use windows::{ ID3D11DeviceContext, ID3D11Texture2D, }, Dxgi::{ - Common::{DXGI_FORMAT, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC}, + Common::{ + DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM, + DXGI_SAMPLE_DESC, + }, IDXGIDevice, }, }, @@ -57,19 +60,22 @@ use windows::{ #[repr(i32)] pub enum PixelFormat { #[default] - R8G8B8A8Unorm = 28, + R8G8B8A8Unorm, + B8G8R8A8Unorm, } impl PixelFormat { pub fn as_directx(&self) -> DirectXPixelFormat { match self { Self::R8G8B8A8Unorm => DirectXPixelFormat::R8G8B8A8UIntNormalized, + Self::B8G8R8A8Unorm => DirectXPixelFormat::B8G8R8A8UIntNormalized, } } pub fn as_dxgi(&self) -> DXGI_FORMAT { match self { Self::R8G8B8A8Unorm => DXGI_FORMAT_R8G8B8A8_UNORM, + Self::B8G8R8A8Unorm => DXGI_FORMAT_B8G8R8A8_UNORM, } } } @@ -134,18 +140,20 @@ pub enum NewCapturerError { } pub struct Capturer { - stop_flag: Arc, - item: GraphicsCaptureItem, settings: Settings, - thread_handle: Option>, d3d_device: ID3D11Device, d3d_context: ID3D11DeviceContext, + session: GraphicsCaptureSession, + _frame_pool: Direct3D11CaptureFramePool, } impl Capturer { pub fn new( item: GraphicsCaptureItem, settings: Settings, + mut callback: impl FnMut(Frame) -> windows::core::Result<()> + Send + 'static, + mut closed_callback: impl FnMut() -> windows::core::Result<()> + Send + 'static, + mut d3d_device: Option, ) -> Result { if !is_supported()? { return Err(NewCapturerError::NotSupported); @@ -165,34 +173,186 @@ impl Capturer { return Err(NewCapturerError::UpdateIntervalNotSupported); } - let mut d3d_device = None; - let mut d3d_context = None; + if d3d_device.is_none() { + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + Default::default(), + None, + D3D11_SDK_VERSION, + Some(&mut d3d_device), + None, + None, + ) + } + .map_err(StartRunnerError::D3DDevice)?; + } - unsafe { - D3D11CreateDevice( - None, - D3D_DRIVER_TYPE_HARDWARE, - HMODULE::default(), - Default::default(), - None, - D3D11_SDK_VERSION, - Some(&mut d3d_device), - None, - Some(&mut d3d_context), - ) + let (d3d_device, d3d_context) = d3d_device + .map(|d| unsafe { d.GetImmediateContext() }.map(|v| (d, v))) + .transpose() + .unwrap() + .unwrap(); + + let item = item.clone(); + let settings = settings.clone(); + let stop_flag = Arc::new(AtomicBool::new(false)); + + let direct3d_device = (|| { + let dxgi_device = d3d_device.cast::()?; + let inspectable = unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi_device) }?; + inspectable.cast::() + })() + .unwrap(); + // .map_err(StartRunnerError::Direct3DDevice)?; + + let frame_pool = Direct3D11CaptureFramePool::CreateFreeThreaded( + &direct3d_device, + settings.pixel_format.as_directx(), + 1, + item.Size().unwrap(), + ) + .unwrap(); + // .map_err(StartRunnerError::FramePool)?; + + let session = frame_pool.CreateCaptureSession(&item).unwrap(); + // .map_err(StartRunnerError::CaptureSession)?; + + if let Some(border_required) = settings.is_border_required { + session.SetIsBorderRequired(border_required).unwrap(); } - .map_err(StartRunnerError::D3DDevice)?; + + if let Some(cursor_capture_enabled) = settings.is_cursor_capture_enabled { + session + .SetIsCursorCaptureEnabled(cursor_capture_enabled) + .unwrap(); + } + + if let Some(min_update_interval) = settings.min_update_interval { + session + .SetMinUpdateInterval(min_update_interval.into()) + .unwrap(); + } + + let crop_data = settings + .crop + .map(|crop| { + let desc = D3D11_TEXTURE2D_DESC { + Width: (crop.right - crop.left), + Height: (crop.bottom - crop.top), + MipLevels: 1, + ArraySize: 1, + Format: settings.pixel_format.as_dxgi(), + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + + let mut texture = None; + unsafe { d3d_device.CreateTexture2D(&desc, None, Some(&mut texture)) } + .map_err(StartRunnerError::CropTexture)?; + + Ok::<_, StartRunnerError>((texture.unwrap(), crop)) + }) + .transpose() + .unwrap(); + + frame_pool + .FrameArrived( + &TypedEventHandler::::new({ + let d3d_context = d3d_context.clone(); + let d3d_device = d3d_device.clone(); + + move |frame_pool, _| { + if stop_flag.load(Ordering::Relaxed) { + return Ok(()); + } + + let frame = frame_pool + .as_ref() + .expect("FrameArrived parameter was None") + .TryGetNextFrame()?; + + let size = frame.ContentSize()?; + + let surface = frame.Surface()?; + let dxgi_interface = surface.cast::()?; + let texture = unsafe { dxgi_interface.GetInterface::() }?; + + let frame = if let Some((cropped_texture, crop)) = crop_data.clone() { + unsafe { + d3d_context.CopySubresourceRegion( + &cropped_texture, + 0, + 0, + 0, + 0, + &texture, + 0, + Some(&crop), + ); + } + + Frame { + width: crop.right - crop.left, + height: crop.bottom - crop.top, + pixel_format: settings.pixel_format, + inner: frame, + texture: cropped_texture, + d3d_context: d3d_context.clone(), + d3d_device: d3d_device.clone(), + } + } else { + Frame { + width: size.Width as u32, + height: size.Height as u32, + pixel_format: settings.pixel_format, + inner: frame, + texture, + d3d_context: d3d_context.clone(), + d3d_device: d3d_device.clone(), + } + }; + + (callback)(frame) + } + }), + ) + .unwrap(); + // .map_err(StartRunnerError::RegisterFrameArrived)?; + + item.Closed( + &TypedEventHandler::::new(move |_, _| { + closed_callback() + }), + ) + .unwrap(); Ok(Capturer { - stop_flag: Arc::new(AtomicBool::new(false)), - item, settings, - thread_handle: None, - d3d_device: d3d_device.unwrap(), - d3d_context: d3d_context.unwrap(), + // thread_handle: None, + d3d_device, + d3d_context, + session, + _frame_pool: frame_pool, }) } + pub fn settings(&self) -> &Settings { + &self.settings + } + + pub fn session(&self) -> &GraphicsCaptureSession { + &self.session + } + pub fn d3d_device(&self) -> &ID3D11Device { &self.d3d_device } @@ -212,62 +372,8 @@ pub enum StartCapturerError { RecvFailed(RecvError), } impl Capturer { - pub fn start( - &mut self, - callback: impl FnMut(Frame) -> windows::core::Result<()> + Send + 'static, - closed_callback: impl FnMut() -> windows::core::Result<()> + Send + 'static, - ) -> Result<(), StartCapturerError> { - if self.thread_handle.is_some() { - return Err(StartCapturerError::AlreadyStarted); - } - - let (started_tx, started_rx) = std::sync::mpsc::channel(); - - let item = self.item.clone(); - let settings = self.settings.clone(); - let stop_flag = self.stop_flag.clone(); - - let d3d_device = self.d3d_device.clone(); - let d3d_context = self.d3d_context.clone(); - - let thread_handle = std::thread::spawn({ - move || { - if let Err(e) = unsafe { RoInitialize(RO_INIT_MULTITHREADED) } - && e.code() != S_FALSE - { - return; - // return Err(CreateRunnerError::FailedToInitializeWinRT); - } - - match Runner::start( - item, - settings, - callback, - closed_callback, - stop_flag, - d3d_device, - d3d_context, - ) { - Ok(runner) => { - let _ = started_tx.send(Ok(())); - - runner.run(); - } - Err(e) => { - let _ = started_tx.send(Err(e)); - } - }; - } - }); - - started_rx - .recv() - .map_err(StartCapturerError::RecvFailed)? - .map_err(StartCapturerError::StartFailed)?; - - self.thread_handle = Some(thread_handle); - - Ok(()) + pub fn start(&mut self) -> windows::core::Result<()> { + self.session.StartCapture() } } @@ -283,44 +389,54 @@ pub enum StopCapturerError { impl Capturer { pub fn stop(&mut self) -> Result<(), StopCapturerError> { - let Some(thread_handle) = self.thread_handle.take() else { - return Err(StopCapturerError::NotStarted); - }; + // let Some(thread_handle) = self.thread_handle.take() else { + // return Err(StopCapturerError::NotStarted); + // }; - self.stop_flag.store(true, Ordering::Relaxed); + // let Some(runner) = self.runner.take() else { + // return Err(StopCapturerError::NotStarted); + // }; - let handle = HANDLE(thread_handle.as_raw_handle()); - let thread_id = unsafe { GetThreadId(handle) }; + // runner._session.Close().unwrap(); - while let Err(e) = - unsafe { PostThreadMessageW(thread_id, WM_QUIT, WPARAM::default(), LPARAM::default()) } - { - if thread_handle.is_finished() { - break; - } + self.session.Close().unwrap(); - if e.code().0 != -2147023452 { - return Err(StopCapturerError::PostMessageFailed); - } - } + // self.runner.self.stop_flag.store(true, Ordering::Relaxed); - thread_handle - .join() - .map_err(|_| StopCapturerError::ThreadJoinFailed) + // let handle = HANDLE(thread_handle.as_raw_handle()); + // let thread_id = unsafe { GetThreadId(handle) }; + + // while let Err(e) = + // unsafe { PostThreadMessageW(thread_id, WM_QUIT, WPARAM::default(), LPARAM::default()) } + // { + // if thread_handle.is_finished() { + // break; + // } + + // if e.code().0 != -2147023452 { + // return Err(StopCapturerError::PostMessageFailed); + // } + // } + + // thread_handle + // .join() + // .map_err(|_| StopCapturerError::ThreadJoinFailed) + + Ok(()) } } -pub struct Frame<'a> { +pub struct Frame { width: u32, height: u32, pixel_format: PixelFormat, inner: Direct3D11CaptureFrame, texture: ID3D11Texture2D, - d3d_device: &'a ID3D11Device, - d3d_context: &'a ID3D11DeviceContext, + d3d_device: ID3D11Device, + d3d_context: ID3D11DeviceContext, } -impl<'a> std::fmt::Debug for Frame<'a> { +impl std::fmt::Debug for Frame { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Frame") .field("width", &self.width) @@ -329,7 +445,7 @@ impl<'a> std::fmt::Debug for Frame<'a> { } } -impl<'a> Frame<'a> { +impl Frame { pub fn width(&self) -> u32 { self.width } @@ -351,14 +467,14 @@ impl<'a> Frame<'a> { } pub fn d3d_device(&self) -> &ID3D11Device { - self.d3d_device + &self.d3d_device } pub fn d3d_context(&self) -> &ID3D11DeviceContext { - self.d3d_context + &self.d3d_context } - pub fn as_buffer(&self) -> windows::core::Result> { + pub fn as_buffer<'a>(&'a self) -> windows::core::Result> { let texture_desc = D3D11_TEXTURE2D_DESC { Width: self.width, Height: self.height, @@ -478,171 +594,37 @@ pub enum StartRunnerError { Other(#[from] windows::core::Error), } -#[derive(Clone)] -struct Runner { - _session: GraphicsCaptureSession, - _frame_pool: Direct3D11CaptureFramePool, -} - -impl Runner { - fn start( - item: GraphicsCaptureItem, - settings: Settings, - mut callback: impl FnMut(Frame) -> windows::core::Result<()> + Send + 'static, - mut closed_callback: impl FnMut() -> windows::core::Result<()> + Send + 'static, - stop_flag: Arc, - d3d_device: ID3D11Device, - d3d_context: ID3D11DeviceContext, - ) -> Result { - let queue_options = DispatcherQueueOptions { - dwSize: std::mem::size_of::() as u32, - threadType: DQTYPE_THREAD_CURRENT, - apartmentType: DQTAT_COM_NONE, - }; - - let _controller = unsafe { CreateDispatcherQueueController(queue_options) } - .map_err(StartRunnerError::DispatchQueue)?; - - let direct3d_device = (|| { - let dxgi_device = d3d_device.cast::()?; - let inspectable = unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi_device) }?; - inspectable.cast::() - })() - .map_err(StartRunnerError::Direct3DDevice)?; - - let frame_pool = Direct3D11CaptureFramePool::Create( - &direct3d_device, - PixelFormat::R8G8B8A8Unorm.as_directx(), - 1, - item.Size()?, - ) - .map_err(StartRunnerError::FramePool)?; - - let session = frame_pool - .CreateCaptureSession(&item) - .map_err(StartRunnerError::CaptureSession)?; - - if let Some(border_required) = settings.is_border_required { - session.SetIsBorderRequired(border_required)?; - } - - if let Some(cursor_capture_enabled) = settings.is_cursor_capture_enabled { - session.SetIsCursorCaptureEnabled(cursor_capture_enabled)?; - } - - if let Some(min_update_interval) = settings.min_update_interval { - session.SetMinUpdateInterval(min_update_interval.into())?; - } - - let crop_data = settings - .crop - .map(|crop| { - let desc = D3D11_TEXTURE2D_DESC { - Width: (crop.right - crop.left), - Height: (crop.bottom - crop.top), - MipLevels: 1, - ArraySize: 1, - Format: settings.pixel_format.as_dxgi(), - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_DEFAULT, - BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, - CPUAccessFlags: 0, - MiscFlags: 0, - }; - - let mut texture = None; - unsafe { d3d_device.CreateTexture2D(&desc, None, Some(&mut texture)) } - .map_err(StartRunnerError::CropTexture)?; - - Ok::<_, StartRunnerError>((texture.unwrap(), crop)) - }) - .transpose()?; - - frame_pool - .FrameArrived( - &TypedEventHandler::::new( - move |frame_pool, _| { - if stop_flag.load(Ordering::Relaxed) { - return Ok(()); - } - - let frame = frame_pool - .as_ref() - .expect("FrameArrived parameter was None") - .TryGetNextFrame()?; - - let size = frame.ContentSize()?; - - let surface = frame.Surface()?; - let dxgi_interface = surface.cast::()?; - let texture = unsafe { dxgi_interface.GetInterface::() }?; - - let frame = if let Some((cropped_texture, crop)) = crop_data.clone() { - unsafe { - d3d_context.CopySubresourceRegion( - &cropped_texture, - 0, - 0, - 0, - 0, - &texture, - 0, - Some(&crop), - ); - } - - Frame { - width: crop.right - crop.left, - height: crop.bottom - crop.top, - pixel_format: settings.pixel_format, - inner: frame, - texture: cropped_texture, - d3d_context: &d3d_context, - d3d_device: &d3d_device, - } - } else { - Frame { - width: size.Width as u32, - height: size.Height as u32, - pixel_format: settings.pixel_format, - inner: frame, - texture, - d3d_context: &d3d_context, - d3d_device: &d3d_device, - } - }; - - (callback)(frame) - }, - ), - ) - .map_err(StartRunnerError::RegisterFrameArrived)?; - - item.Closed( - &TypedEventHandler::::new(move |_, _| { - closed_callback() - }), - ) - .map_err(StartRunnerError::RegisterClosed)?; - - session - .StartCapture() - .map_err(StartRunnerError::StartCapture)?; - - Ok(Self { - _session: session, - _frame_pool: frame_pool, - }) - } - - fn run(self) { - let mut message = MSG::default(); - while unsafe { GetMessageW(&mut message, None, 0, 0) }.as_bool() { - let _ = unsafe { TranslateMessage(&message) }; - unsafe { DispatchMessageW(&message) }; - } - } -} +// #[derive(Clone)] +// struct Runner { +// _session: GraphicsCaptureSession, +// _frame_pool: Direct3D11CaptureFramePool, +// } + +// impl Runner { +// fn start( +// item: GraphicsCaptureItem, +// settings: Settings, +// mut callback: impl FnMut(Frame) -> windows::core::Result<()> + Send + 'static, +// mut closed_callback: impl FnMut() -> windows::core::Result<()> + Send + 'static, +// stop_flag: Arc, +// d3d_device: ID3D11Device, +// d3d_context: ID3D11DeviceContext, +// ) -> Result { +// session +// .StartCapture() +// .map_err(StartRunnerError::StartCapture)?; + +// Ok(Self { +// _session: session, +// _frame_pool: frame_pool, +// }) +// } + +// // fn run(self) { +// // let mut message = MSG::default(); +// // while unsafe { GetMessageW(&mut message, None, 0, 0) }.as_bool() { +// // let _ = unsafe { TranslateMessage(&message) }; +// // unsafe { DispatchMessageW(&message) }; +// // } +// // } +// } diff --git a/crates/scap-ffmpeg/Cargo.toml b/crates/scap-ffmpeg/Cargo.toml index a9ec90ece5..4cfee27bda 100644 --- a/crates/scap-ffmpeg/Cargo.toml +++ b/crates/scap-ffmpeg/Cargo.toml @@ -6,9 +6,9 @@ license = "MIT" [dependencies] ffmpeg = { workspace = true } - -scap-cpal = { optional = true, path = "../scap-cpal" } cpal = { workspace = true } +scap-cpal = { optional = true, path = "../scap-cpal" } +cap-ffmpeg-utils = { path = "../ffmpeg-utils" } [target.'cfg(windows)'.dependencies] scap-direct3d = { path = "../scap-direct3d" } diff --git a/crates/scap-ffmpeg/src/cpal.rs b/crates/scap-ffmpeg/src/cpal.rs index cabbfc0921..1da8a8b1bd 100644 --- a/crates/scap-ffmpeg/src/cpal.rs +++ b/crates/scap-ffmpeg/src/cpal.rs @@ -1,3 +1,4 @@ +use cap_ffmpeg_utils::PlanarData; use cpal::{SampleFormat, StreamConfig}; use ffmpeg::format::{Sample, sample}; @@ -43,39 +44,3 @@ impl DataExt for ::cpal::Data { ffmpeg_frame } } - -pub trait PlanarData { - fn plane_data(&self, index: usize) -> &[u8]; - - fn plane_data_mut(&mut self, index: usize) -> &mut [u8]; -} - -impl PlanarData for ffmpeg::frame::Audio { - #[inline] - fn plane_data(&self, index: usize) -> &[u8] { - if index >= self.planes() { - panic!("out of bounds"); - } - - unsafe { - std::slice::from_raw_parts( - (*self.as_ptr()).data[index], - (*self.as_ptr()).linesize[0] as usize, - ) - } - } - - #[inline] - fn plane_data_mut(&mut self, index: usize) -> &mut [u8] { - if index >= self.planes() { - panic!("out of bounds"); - } - - unsafe { - std::slice::from_raw_parts_mut( - (*self.as_mut_ptr()).data[index], - (*self.as_ptr()).linesize[0] as usize, - ) - } - } -} diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 9c2985d6b3..0aec041f02 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -3,7 +3,7 @@ use scap_direct3d::PixelFormat; pub type AsFFmpegError = windows::core::Error; -impl<'a> super::AsFFmpeg for scap_direct3d::Frame<'a> { +impl super::AsFFmpeg for scap_direct3d::Frame { fn as_ffmpeg(&self) -> Result { let buffer = self.as_buffer()?; @@ -33,6 +33,27 @@ impl<'a> super::AsFFmpeg for scap_direct3d::Frame<'a> { dest_row.copy_from_slice(src_row); } + Ok(ff_frame) + } + PixelFormat::B8G8R8A8Unorm => { + let mut ff_frame = ffmpeg::frame::Video::new( + ffmpeg::format::Pixel::BGRA, + self.width(), + self.height(), + ); + + let dest_stride = ff_frame.stride(0); + let dest_bytes = ff_frame.data_mut(0); + + let row_length = width * 4; + + for i in 0..height { + let src_row = &src_bytes[i * src_stride..i * src_stride + row_length]; + let dest_row = &mut dest_bytes[i * dest_stride..i * dest_stride + row_length]; + + dest_row.copy_from_slice(src_row); + } + Ok(ff_frame) } } @@ -47,6 +68,7 @@ impl PixelFormatExt for PixelFormat { fn as_ffmpeg(&self) -> Pixel { match self { PixelFormat::R8G8B8A8Unorm => Pixel::RGBA, + PixelFormat::B8G8R8A8Unorm => Pixel::BGRA, } } } diff --git a/crates/scap-screencapturekit/examples/cli.rs b/crates/scap-screencapturekit/examples/cli.rs index f0b3e78c5b..36d95dbc2a 100644 --- a/crates/scap-screencapturekit/examples/cli.rs +++ b/crates/scap-screencapturekit/examples/cli.rs @@ -1,46 +1,55 @@ -use scap_targets::Display; -use std::time::Duration; - -use futures::executor::block_on; -use scap_screencapturekit::{Capturer, StreamCfgBuilder}; - fn main() { - let display = Display::primary(); - let display = display.raw_handle(); - - // let windows = block_on(Window::list()).expect("Failed to list windows"); - // let window = windows - // .iter() - // .find(|w| w.title().map(|t| t.starts_with("native")).unwrap_or(false)) - // .expect("No native window found"); - - let config = StreamCfgBuilder::default() - .with_fps(60.0) - .with_width(display.physical_size().width() as usize) - .with_height(display.physical_size().height() as usize) - .build(); - - let capturer = Capturer::builder( - block_on(display.as_content_filter()).expect("Failed to get display as content filter"), - config, - ) - .with_output_sample_buf_cb(|frame| { - dbg!(frame.output_type()); - // if let Some(image_buf) = buf.image_buf() { - // image_buf.show(); - // } - }) - .with_stop_with_err_cb(|stream, error| { - dbg!(stream, error); - }) - .build() - .expect("Failed to build capturer"); - - block_on(capturer.start()).expect("Failed to start capturing"); - - std::thread::sleep(Duration::from_secs(3)); - - block_on(capturer.stop()).expect("Failed to stop capturing"); - - std::thread::sleep(Duration::from_secs(1)); + #[cfg(target_os = "macos")] + macos::main(); +} + +#[cfg(target_os = "macos")] +mod macos { + + use scap_targets::Display; + use std::time::Duration; + + use futures::executor::block_on; + use scap_screencapturekit::{Capturer, StreamCfgBuilder}; + + fn main() { + let display = Display::primary(); + let display = display.raw_handle(); + + // let windows = block_on(Window::list()).expect("Failed to list windows"); + // let window = windows + // .iter() + // .find(|w| w.title().map(|t| t.starts_with("native")).unwrap_or(false)) + // .expect("No native window found"); + + let config = StreamCfgBuilder::default() + .with_fps(60.0) + .with_width(display.physical_size().width() as usize) + .with_height(display.physical_size().height() as usize) + .build(); + + let capturer = Capturer::builder( + block_on(display.as_content_filter()).expect("Failed to get display as content filter"), + config, + ) + .with_output_sample_buf_cb(|frame| { + dbg!(frame.output_type()); + // if let Some(image_buf) = buf.image_buf() { + // image_buf.show(); + // } + }) + .with_stop_with_err_cb(|stream, error| { + dbg!(stream, error); + }) + .build() + .expect("Failed to build capturer"); + + block_on(capturer.start()).expect("Failed to start capturing"); + + std::thread::sleep(Duration::from_secs(3)); + + block_on(capturer.stop()).expect("Failed to stop capturing"); + + std::thread::sleep(Duration::from_secs(1)); + } } diff --git a/crates/scap-targets/Cargo.toml b/crates/scap-targets/Cargo.toml index 0809ad1342..4071af147d 100644 --- a/crates/scap-targets/Cargo.toml +++ b/crates/scap-targets/Cargo.toml @@ -21,11 +21,13 @@ objc = "0.2.7" [target.'cfg(target_os = "windows")'.dependencies] windows = { workspace = true, features = [ + "Graphics_Capture", "Win32_Foundation", "Win32_System_Threading", "Win32_UI_WindowsAndMessaging", "Win32_UI_Shell", "Win32_UI_HiDpi", + "Win32_Graphics_Dwm", "Win32_Graphics_Gdi", "Win32_Storage_FileSystem", ] } diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index ff67bd3fcf..424407d67e 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -38,17 +38,17 @@ declare global { const IconCapLogoFullDark: typeof import('~icons/cap/logo-full-dark.jsx')['default'] const IconCapMessageBubble: typeof import('~icons/cap/message-bubble.jsx')['default'] const IconCapMicrophone: typeof import('~icons/cap/microphone.jsx')['default'] - const IconCapMoreVertical: typeof import("~icons/cap/more-vertical.jsx")["default"] + const IconCapMoreVertical: typeof import('~icons/cap/more-vertical.jsx')['default'] const IconCapNext: typeof import('~icons/cap/next.jsx')['default'] const IconCapPadding: typeof import('~icons/cap/padding.jsx')['default'] const IconCapPause: typeof import('~icons/cap/pause.jsx')['default'] - const IconCapPauseCircle: typeof import("~icons/cap/pause-circle.jsx")["default"] + const IconCapPauseCircle: typeof import('~icons/cap/pause-circle.jsx')['default'] const IconCapPlay: typeof import('~icons/cap/play.jsx')['default'] - const IconCapPlayCircle: typeof import("~icons/cap/play-circle.jsx")["default"] + const IconCapPlayCircle: typeof import('~icons/cap/play-circle.jsx')['default'] const IconCapPresets: typeof import('~icons/cap/presets.jsx')['default'] const IconCapPrev: typeof import('~icons/cap/prev.jsx')['default'] const IconCapRedo: typeof import('~icons/cap/redo.jsx')['default'] - const IconCapRestart: typeof import("~icons/cap/restart.jsx")["default"] + const IconCapRestart: typeof import('~icons/cap/restart.jsx')['default'] const IconCapScissors: typeof import('~icons/cap/scissors.jsx')['default'] const IconCapSettings: typeof import('~icons/cap/settings.jsx')['default'] const IconCapShadow: typeof import('~icons/cap/shadow.jsx')['default'] @@ -76,7 +76,7 @@ declare global { const IconLucideLayout: typeof import('~icons/lucide/layout.jsx')['default'] const IconLucideLoaderCircle: typeof import('~icons/lucide/loader-circle.jsx')['default'] const IconLucideMessageSquarePlus: typeof import('~icons/lucide/message-square-plus.jsx')['default'] - const IconLucideMicOff: typeof import("~icons/lucide/mic-off.jsx")["default"] + const IconLucideMicOff: typeof import('~icons/lucide/mic-off.jsx')['default'] const IconLucideMonitor: typeof import('~icons/lucide/monitor.jsx')['default'] const IconLucideRectangleHorizontal: typeof import('~icons/lucide/rectangle-horizontal.jsx')['default'] const IconLucideRotateCcw: typeof import('~icons/lucide/rotate-ccw.jsx')['default']