From 1ff8bf346aab4d0d93da1bc1c32cc5230ded142a Mon Sep 17 00:00:00 2001 From: ronen barzel Date: Tue, 1 Aug 2023 08:24:29 -0400 Subject: [PATCH] Factor out media device query and selection (#112) * factor out model::media_access::MediaAccess ...and tidy up AttendantsCommponent::create() * factor MediaDevicesList out of DeviceSelector component * cargo fmt * use AtomicBool instead of Cell * cargo fmt * bug fix: was calling on_granted() when permission was denied. --------- Co-authored-by: Dario A Lencina-Talarico --- yew-ui/src/components/attendants.rs | 93 +++++++------ yew-ui/src/components/device_permissions.rs | 23 ---- yew-ui/src/components/device_selector.rs | 122 ++++++------------ yew-ui/src/components/mod.rs | 1 - yew-ui/src/main.rs | 1 + .../media_devices/media_device_access.rs | 62 +++++++++ .../model/media_devices/media_device_list.rs | 117 +++++++++++++++++ yew-ui/src/model/media_devices/mod.rs | 5 + yew-ui/src/model/mod.rs | 1 + 9 files changed, 279 insertions(+), 146 deletions(-) delete mode 100644 yew-ui/src/components/device_permissions.rs create mode 100644 yew-ui/src/model/media_devices/media_device_access.rs create mode 100644 yew-ui/src/model/media_devices/media_device_list.rs create mode 100644 yew-ui/src/model/media_devices/mod.rs diff --git a/yew-ui/src/components/attendants.rs b/yew-ui/src/components/attendants.rs index aeb58789..a030a624 100644 --- a/yew-ui/src/components/attendants.rs +++ b/yew-ui/src/components/attendants.rs @@ -3,6 +3,7 @@ use std::rc::Rc; use crate::constants::WEBTRANSPORT_HOST; use crate::model::connection::{ConnectOptions, Connection}; use crate::model::decode::PeerDecodeManager; +use crate::model::media_devices::MediaDeviceAccess; use crate::model::MediaPacketWrapper; use crate::{components::host::Host, constants::ACTIX_WEBSOCKET}; use gloo_console::log; @@ -11,14 +12,12 @@ use types::protos::media_packet::MediaPacket; use wasm_bindgen::JsCast; use wasm_bindgen::JsValue; +use super::icons::push_pin::PushPinIcon; use web_sys::*; use yew::prelude::*; use yew::virtual_dom::VNode; use yew::{html, Component, Context, Html}; -use super::device_permissions::request_permissions; -use super::icons::push_pin::PushPinIcon; - #[derive(Debug)] pub enum WsAction { Connect(bool), @@ -72,43 +71,72 @@ pub struct AttendantsComponentProps { pub struct AttendantsComponent { pub connection: Option, pub peer_decode_manager: PeerDecodeManager, + pub media_device_access: MediaDeviceAccess, pub outbound_audio_buffer: [u8; 2000], pub share_screen: bool, pub webtransport_enabled: bool, pub mic_enabled: bool, pub video_enabled: bool, pub error: Option, - pub media_access_granted: bool, } -impl Component for AttendantsComponent { - type Message = Msg; - type Properties = AttendantsComponentProps; +impl AttendantsComponent { + fn is_connected(&self) -> bool { + match &self.connection { + Some(connection) => connection.is_connected(), + None => false, + } + } - fn create(ctx: &Context) -> Self { - let webtransport_enabled = ctx.props().webtransport_enabled; + fn create_peer_decoder_manager(ctx: &Context) -> PeerDecodeManager { let mut peer_decode_manager = PeerDecodeManager::new(); - let link = ctx.link().clone(); - peer_decode_manager.on_peer_added = Callback::from(move |email| { - link.send_message(Msg::OnPeerAdded(email)); - }); - let link = ctx.link().clone(); - peer_decode_manager.on_first_frame = Callback::from(move |(email, media_type)| { - link.send_message(Msg::OnFirstFrame((email, media_type))); - }); + peer_decode_manager.on_peer_added = { + let link = ctx.link().clone(); + Callback::from(move |email| link.send_message(Msg::OnPeerAdded(email))) + }; + peer_decode_manager.on_first_frame = { + let link = ctx.link().clone(); + Callback::from(move |(email, media_type)| { + link.send_message(Msg::OnFirstFrame((email, media_type))) + }) + }; peer_decode_manager.get_video_canvas_id = Callback::from(|email| email); peer_decode_manager.get_screen_canvas_id = Callback::from(|email| format!("screen-share-{}", &email)); + peer_decode_manager + } + + fn create_media_device_access(ctx: &Context) -> MediaDeviceAccess { + let mut media_device_access = MediaDeviceAccess::new(); + media_device_access.on_granted = { + let link = ctx.link().clone(); + Callback::from(move |_| link.send_message(WsAction::MediaPermissionsGranted)) + }; + media_device_access.on_denied = { + let link = ctx.link().clone(); + Callback::from(move |_| { + link.send_message(WsAction::MediaPermissionsError("Error requesting permissions. Please make sure to allow access to both camera and microphone.".to_string())) + }) + }; + media_device_access + } +} + +impl Component for AttendantsComponent { + type Message = Msg; + type Properties = AttendantsComponentProps; + + fn create(ctx: &Context) -> Self { Self { connection: None, - peer_decode_manager, + peer_decode_manager: Self::create_peer_decoder_manager(ctx), + media_device_access: Self::create_media_device_access(ctx), outbound_audio_buffer: [0; 2000], share_screen: false, mic_enabled: false, video_enabled: false, - webtransport_enabled, + webtransport_enabled: ctx.props().webtransport_enabled, error: None, - media_access_granted: false, } } @@ -166,23 +194,11 @@ impl Component for AttendantsComponent { true } WsAction::RequestMediaPermissions => { - let future = request_permissions(); - let link = ctx.link().clone(); - wasm_bindgen_futures::spawn_local(async move { - match future.await { - Ok(_) => { - link.send_message(WsAction::MediaPermissionsGranted); - } - Err(_) => { - link.send_message(WsAction::MediaPermissionsError("Error requesting permissions. Please make sure to allow access to both camera and microphone.".to_string())); - } - } - }); + self.media_device_access.request(); false } WsAction::MediaPermissionsGranted => { self.error = None; - self.media_access_granted = true; ctx.link() .send_message(WsAction::Connect(self.webtransport_enabled)); true @@ -229,7 +245,7 @@ impl Component for AttendantsComponent { fn view(&self, ctx: &Context) -> Html { let email = ctx.props().email.clone(); let on_packet = ctx.link().callback(Msg::OnOutboundPacket); - let media_access_granted = self.media_access_granted; + let media_access_granted = self.media_device_access.is_granted(); let rows: Vec = self .peer_decode_manager .sorted_keys() @@ -320,15 +336,6 @@ impl Component for AttendantsComponent { } } -impl AttendantsComponent { - fn is_connected(&self) -> bool { - match &self.connection { - Some(connection) => connection.is_connected(), - None => false, - } - } -} - // props for the video component #[derive(Properties, Debug, PartialEq)] pub struct UserVideoProps { diff --git a/yew-ui/src/components/device_permissions.rs b/yew-ui/src/components/device_permissions.rs deleted file mode 100644 index a6ce6307..00000000 --- a/yew-ui/src/components/device_permissions.rs +++ /dev/null @@ -1,23 +0,0 @@ -use gloo_utils::window; -use wasm_bindgen::prelude::*; -use wasm_bindgen_futures::JsFuture; -use web_sys::MediaStreamConstraints; - -pub async fn request_permissions() -> anyhow::Result<(), JsValue> { - let navigator = window().navigator(); - let media_devices = navigator.media_devices()?; - - let mut constraints = MediaStreamConstraints::new(); - - // Request access to the microphone - constraints.audio(&JsValue::from_bool(true)); - - // Request access to the camera - constraints.video(&JsValue::from_bool(true)); - - let promise = media_devices.get_user_media_with_constraints(&constraints)?; - - JsFuture::from(promise).await?; - - Ok(()) -} diff --git a/yew-ui/src/components/device_selector.rs b/yew-ui/src/components/device_selector.rs index cbb621ff..584750c9 100644 --- a/yew-ui/src/components/device_selector.rs +++ b/yew-ui/src/components/device_selector.rs @@ -1,23 +1,14 @@ -use gloo_utils::window; -use js_sys::Array; -use js_sys::Promise; +use crate::model::media_devices::MediaDeviceList; use wasm_bindgen::JsCast; -use wasm_bindgen_futures::JsFuture; -use web_sys::EventTarget; use web_sys::HtmlSelectElement; -use web_sys::MediaDeviceInfo; -use web_sys::MediaDeviceKind; use yew::prelude::*; pub struct DeviceSelector { - audio_devices: Vec, - video_devices: Vec, - video_selected: Option, - audio_selected: Option, + media_devices: MediaDeviceList, } pub enum Msg { - DevicesLoaded(Vec), + DevicesLoaded, OnCameraSelect(String), OnMicSelect(String), LoadDevices(), @@ -29,6 +20,21 @@ pub struct DeviceSelectorProps { pub on_microphone_select: Callback, } +impl DeviceSelector { + fn create_media_device_list(ctx: &Context) -> MediaDeviceList { + let mut media_devices = MediaDeviceList::new(); + let link = ctx.link().clone(); + let on_microphone_select = ctx.props().on_microphone_select.clone(); + let on_camera_select = ctx.props().on_camera_select.clone(); + media_devices.on_loaded = Callback::from(move |_| link.send_message(Msg::DevicesLoaded)); + media_devices.audio_inputs.on_selected = + Callback::from(move |device_id| on_microphone_select.emit(device_id)); + media_devices.video_inputs.on_selected = + Callback::from(move |device_id| on_camera_select.emit(device_id)); + media_devices + } +} + impl Component for DeviceSelector { type Message = Msg; type Properties = DeviceSelectorProps; @@ -39,10 +45,7 @@ impl Component for DeviceSelector { link.send_message(Msg::LoadDevices()); }); Self { - audio_devices: Vec::new(), - video_devices: Vec::new(), - audio_selected: None, - video_selected: None, + media_devices: Self::create_media_device_list(ctx), } } @@ -52,79 +55,44 @@ impl Component for DeviceSelector { } } - fn update(&mut self, ctx: &Context, msg: Self::Message) -> bool { + fn update(&mut self, _ctx: &Context, msg: Self::Message) -> bool { match msg { Msg::LoadDevices() => { - let link = ctx.link().clone(); - wasm_bindgen_futures::spawn_local(async move { - let navigator = window().navigator(); - let media_devices = navigator.media_devices().unwrap(); - - let promise: Promise = media_devices - .enumerate_devices() - .expect("enumerate devices"); - let future = JsFuture::from(promise); - let devices = future - .await - .expect("await devices") - .unchecked_into::(); - let devices = devices.to_vec(); - let devices = devices - .into_iter() - .map(|d| d.unchecked_into::()) - .collect::>(); - link.send_message(Msg::DevicesLoaded(devices)); - }); + self.media_devices.load(); false } - Msg::DevicesLoaded(devices) => { - self.audio_devices = devices - .clone() - .into_iter() - .filter(|device| device.kind() == MediaDeviceKind::Audioinput) - .collect(); - self.video_devices = devices - .into_iter() - .filter(|device| device.kind() == MediaDeviceKind::Videoinput) - .collect(); - ctx.props() - .on_camera_select - .emit(self.video_devices[0].device_id()); - ctx.props() - .on_microphone_select - .emit(self.audio_devices[0].device_id()); - self.video_selected = Some(self.video_devices[0].device_id()); - self.audio_selected = Some(self.audio_devices[0].device_id()); - true - } + Msg::DevicesLoaded => true, Msg::OnCameraSelect(camera) => { - ctx.props().on_camera_select.emit(camera.clone()); - self.video_selected = Some(camera); + self.media_devices.video_inputs.select(&camera); true } Msg::OnMicSelect(mic) => { - ctx.props().on_microphone_select.emit(mic.clone()); - self.audio_selected = Some(mic); + self.media_devices.audio_inputs.select(&mic); true } } } fn view(&self, ctx: &Context) -> Html { - let selected_mic = self.audio_selected.clone().unwrap_or_default(); - let selected_camera = self.video_selected.clone().unwrap_or_default(); + let mics = self.media_devices.audio_inputs.devices(); + let cameras = self.media_devices.video_inputs.devices(); + let selected_mic = self.media_devices.audio_inputs.selected(); + let selected_camera = self.media_devices.video_inputs.selected(); + fn selection(event: Event) -> String { + event + .target() + .expect("Event should have a target when dispatched") + .unchecked_into::() + .value() + } + html! {

- + { for cameras.iter().map(|device| html! { diff --git a/yew-ui/src/components/mod.rs b/yew-ui/src/components/mod.rs index beb24cad..5bbde77e 100644 --- a/yew-ui/src/components/mod.rs +++ b/yew-ui/src/components/mod.rs @@ -1,5 +1,4 @@ pub mod attendants; -pub mod device_permissions; pub mod device_selector; pub mod host; pub mod icons; diff --git a/yew-ui/src/main.rs b/yew-ui/src/main.rs index 1a3f96ae..1b36e16e 100644 --- a/yew-ui/src/main.rs +++ b/yew-ui/src/main.rs @@ -1,4 +1,5 @@ #![feature(future_join)] +#![feature(once_cell)] mod components; mod constants; diff --git a/yew-ui/src/model/media_devices/media_device_access.rs b/yew-ui/src/model/media_devices/media_device_access.rs new file mode 100644 index 00000000..5709344d --- /dev/null +++ b/yew-ui/src/model/media_devices/media_device_access.rs @@ -0,0 +1,62 @@ +use gloo_utils::window; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; +use web_sys::MediaStreamConstraints; +use yew::prelude::Callback; + +pub struct MediaDeviceAccess { + granted: Arc, + pub on_granted: Callback<()>, + pub on_denied: Callback<()>, +} + +impl MediaDeviceAccess { + pub fn new() -> Self { + Self { + granted: Arc::new(AtomicBool::new(false)), + on_granted: Callback::noop(), + on_denied: Callback::noop(), + } + } + + pub fn is_granted(&self) -> bool { + self.granted.load(Ordering::Acquire) + } + + pub fn request(&self) { + let future = Self::request_permissions(); + let on_granted = self.on_granted.clone(); + let on_denied = self.on_denied.clone(); + let granted = Arc::clone(&self.granted); + wasm_bindgen_futures::spawn_local(async move { + match future.await { + Ok(_) => { + granted.store(true, Ordering::Release); + on_granted.emit(()); + } + Err(_) => on_denied.emit(()), + } + }); + } + + async fn request_permissions() -> anyhow::Result<(), JsValue> { + let navigator = window().navigator(); + let media_devices = navigator.media_devices()?; + + let mut constraints = MediaStreamConstraints::new(); + + // Request access to the microphone + constraints.audio(&JsValue::from_bool(true)); + + // Request access to the camera + constraints.video(&JsValue::from_bool(true)); + + let promise = media_devices.get_user_media_with_constraints(&constraints)?; + + JsFuture::from(promise).await?; + + Ok(()) + } +} diff --git a/yew-ui/src/model/media_devices/media_device_list.rs b/yew-ui/src/model/media_devices/media_device_list.rs new file mode 100644 index 00000000..41d1bed9 --- /dev/null +++ b/yew-ui/src/model/media_devices/media_device_list.rs @@ -0,0 +1,117 @@ +use gloo_utils::window; +use js_sys::Array; +use js_sys::Promise; +use std::cell::OnceCell; +use std::sync::Arc; +use wasm_bindgen::JsCast; +use wasm_bindgen_futures::JsFuture; +use web_sys::MediaDeviceInfo; +use web_sys::MediaDeviceKind; +use yew::prelude::Callback; + +pub struct SelectableDevices { + devices: Arc>>, + selected: Option, + pub on_selected: Callback, +} + +impl SelectableDevices { + fn new() -> Self { + Self { + devices: Arc::new(OnceCell::new()), + selected: None, + on_selected: Callback::noop(), + } + } + + pub fn select(&mut self, device_id: &str) { + if let Some(devices) = self.devices.get() { + for device in devices.iter() { + if device.device_id() == device_id { + self.selected = Some(device_id.to_string()); + self.on_selected.emit(device_id.to_string()); + } + } + } + } + + pub fn devices(&self) -> &[MediaDeviceInfo] { + match self.devices.get() { + Some(devices) => devices, + None => &[], + } + } + + pub fn selected(&self) -> String { + match &self.selected { + Some(selected) => selected.to_string(), + // device 0 is the default selection + None => match self.devices().get(0) { + Some(device) => device.device_id(), + None => "".to_string(), + }, + } + } +} + +pub struct MediaDeviceList { + pub audio_inputs: SelectableDevices, + pub video_inputs: SelectableDevices, + pub on_loaded: Callback<()>, +} + +impl MediaDeviceList { + pub fn new() -> Self { + Self { + audio_inputs: SelectableDevices::new(), + video_inputs: SelectableDevices::new(), + on_loaded: Callback::noop(), + } + } + + pub fn load(&self) { + let on_loaded = self.on_loaded.clone(); + let on_audio_selected = self.audio_inputs.on_selected.clone(); + let on_video_selected = self.video_inputs.on_selected.clone(); + let audio_input_devices = Arc::clone(&self.audio_inputs.devices); + let video_input_devices = Arc::clone(&self.video_inputs.devices); + wasm_bindgen_futures::spawn_local(async move { + let navigator = window().navigator(); + let media_devices = navigator.media_devices().unwrap(); + + let promise: Promise = media_devices + .enumerate_devices() + .expect("enumerate devices"); + let future = JsFuture::from(promise); + let devices = future + .await + .expect("await devices") + .unchecked_into::(); + let devices = devices.to_vec(); + let devices = devices + .into_iter() + .map(|d| d.unchecked_into::()) + .collect::>(); + _ = audio_input_devices.set( + devices + .clone() + .into_iter() + .filter(|device| device.kind() == MediaDeviceKind::Audioinput) + .collect(), + ); + _ = video_input_devices.set( + devices + .into_iter() + .filter(|device| device.kind() == MediaDeviceKind::Videoinput) + .collect(), + ); + if let Some(device) = audio_input_devices.get().unwrap().get(0) { + on_audio_selected.emit(device.device_id()) + } + if let Some(device) = video_input_devices.get().unwrap().get(0) { + on_video_selected.emit(device.device_id()) + } + on_loaded.emit(()); + }); + } +} diff --git a/yew-ui/src/model/media_devices/mod.rs b/yew-ui/src/model/media_devices/mod.rs new file mode 100644 index 00000000..957b0c25 --- /dev/null +++ b/yew-ui/src/model/media_devices/mod.rs @@ -0,0 +1,5 @@ +mod media_device_access; +mod media_device_list; + +pub use media_device_access::MediaDeviceAccess; +pub use media_device_list::MediaDeviceList; diff --git a/yew-ui/src/model/mod.rs b/yew-ui/src/model/mod.rs index e6ac5547..36b4dd0f 100644 --- a/yew-ui/src/model/mod.rs +++ b/yew-ui/src/model/mod.rs @@ -1,6 +1,7 @@ pub mod connection; pub mod decode; pub mod encode; +pub mod media_devices; pub mod wrappers; pub use wrappers::{