From 5bb04454821f30acfb951f0facaa75cf48ddf474 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 23 Feb 2023 23:49:49 +0800 Subject: [PATCH 01/11] fix: Force audio settings to be consistent. --- lib/src/hardware/hardware.dart | 28 +++++++++++++++++++++++++++- lib/src/participant/local.dart | 1 + lib/src/track/audio_management.dart | 7 +++++++ lib/src/track/local/local.dart | 5 +++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/src/hardware/hardware.dart b/lib/src/hardware/hardware.dart index a55c774d..8d2fedf9 100644 --- a/lib/src/hardware/hardware.dart +++ b/lib/src/hardware/hardware.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'package:collection/collection.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:meta/meta.dart'; class MediaDevice { const MediaDevice(this.deviceId, this.label, this.kind); @@ -40,6 +41,8 @@ class Hardware { devices.firstWhereOrNull((element) => element.kind == 'audiooutput'); selectedVideoInput ??= devices.firstWhereOrNull((element) => element.kind == 'videoinput'); + _lastDevices = devices; + speakerOn = true; }); } @@ -54,6 +57,10 @@ class Hardware { MediaDevice? selectedVideoInput; + bool? speakerOn; + + List? _lastDevices; + Future> enumerateDevices({String? type}) async { var infos = await rtc.navigator.mediaDevices.enumerateDevices(); var devices = @@ -85,7 +92,7 @@ class Hardware { } Future selectAudioInput(MediaDevice device) async { - if (rtc.WebRTC.platformIsWeb || rtc.WebRTC.platformIsIOS) { + if (rtc.WebRTC.platformIsWeb) { throw UnimplementedError( 'selectAudioInput is only supported on Android/Windows/macOS'); } @@ -95,6 +102,7 @@ class Hardware { Future setSpeakerphoneOn(bool enable) async { if (rtc.WebRTC.platformIsMobile) { + speakerOn = enable; await rtc.Helper.setSpeakerphoneOn(enable); } else { throw UnimplementedError('setSpeakerphoneOn only support on iOS/Android'); @@ -131,5 +139,23 @@ class Hardware { selectedVideoInput ??= devices.firstWhereOrNull((element) => element.kind == 'videoinput'); onDeviceChange.add(devices); + _lastDevices = devices; + } + + @internal + Future applyAudioSettings() async { + var devices = await enumerateDevices(); + // if devices no changes, reselect audio input/output + if (_lastDevices?.equals(devices) == true) { + if (selectedAudioInput != null && !rtc.WebRTC.platformIsWeb) { + await selectAudioInput(selectedAudioInput!); + } + if (selectedAudioOutput != null && !rtc.WebRTC.platformIsWeb) { + await selectAudioOutput(selectedAudioOutput!); + } + } + if (speakerOn != null && rtc.WebRTC.platformIsMobile) { + await setSpeakerphoneOn(speakerOn!); + } } } diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index e4ce1e26..4e89158f 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -319,6 +319,7 @@ class LocalParticipant extends Participant { await publication.mute(); } } + await publication.track?.applyAudioSettings(); return publication; } else if (enabled) { if (source == TrackSource.camera) { diff --git a/lib/src/track/audio_management.dart b/lib/src/track/audio_management.dart index f303e374..ff512a96 100644 --- a/lib/src/track/audio_management.dart +++ b/lib/src/track/audio_management.dart @@ -1,3 +1,4 @@ +import 'package:livekit_client/src/hardware/hardware.dart'; import 'package:synchronized/synchronized.dart' as sync; import '../logger.dart'; @@ -27,6 +28,12 @@ int _localTrackCount = 0; int _remoteTrackCount = 0; mixin LocalAudioManagementMixin on LocalTrack, AudioTrack { + @override + Future applyAudioSettings() async { + var hardware = Hardware.instance; + await hardware.applyAudioSettings(); + } + @override Future onPublish() async { final didUpdate = await super.onPublish(); diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index c8dceb74..e2742bcb 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -183,6 +183,11 @@ abstract class LocalTrack extends Track { )); } + @internal + Future applyAudioSettings() async { + logger.fine('$objectId.applyAudioSettings()'); + } + @internal @mustCallSuper Future onPublish() async { From 4c1014a054b95d23447bef046e28e8eb50faf9f4 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 23 Feb 2023 23:52:45 +0800 Subject: [PATCH 02/11] chore: applyAudioSettings for onUnpublish. --- lib/src/participant/local.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index 4e89158f..9b10e025 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -88,6 +88,7 @@ class LocalParticipant extends Participant { // did publish await track.onPublish(); + await track.applyAudioSettings(); [events, room.events].emit(LocalTrackPublishedEvent( participant: this, @@ -228,6 +229,7 @@ class LocalParticipant extends Participant { // did unpublish await track.onUnpublish(); + await track.applyAudioSettings(); } if (notify) { From c7f814ba3994bfa796a3273fe56fa77c72bf715e Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 14:50:50 +0800 Subject: [PATCH 03/11] revert audio output select for web. --- example/lib/widgets/controls.dart | 72 ++++++++++++++-------------- example/lib/widgets/participant.dart | 70 ++------------------------- 2 files changed, 40 insertions(+), 102 deletions(-) diff --git a/example/lib/widgets/controls.dart b/example/lib/widgets/controls.dart index 2da90b1e..5c2256c6 100644 --- a/example/lib/widgets/controls.dart +++ b/example/lib/widgets/controls.dart @@ -317,45 +317,43 @@ class _ControlsWidgetState extends State { icon: const Icon(EvaIcons.micOff), tooltip: 'un-mute audio', ), - if (!lkPlatformIs(PlatformType.web)) - PopupMenuButton( - icon: const Icon(Icons.volume_up), - itemBuilder: (BuildContext context) { - return [ - const PopupMenuItem( - value: null, - child: ListTile( - leading: Icon( - EvaIcons.speaker, - color: Colors.white, - ), - title: Text('Select Audio Output'), + PopupMenuButton( + icon: const Icon(Icons.volume_up), + itemBuilder: (BuildContext context) { + return [ + const PopupMenuItem( + value: null, + child: ListTile( + leading: Icon( + EvaIcons.speaker, + color: Colors.white, ), + title: Text('Select Audio Output'), ), - if (_audioOutputs != null) - ..._audioOutputs!.map((device) { - return PopupMenuItem( - value: device, - child: ListTile( - leading: (device.deviceId == - Hardware - .instance.selectedAudioOutput?.deviceId) - ? const Icon( - EvaIcons.checkmarkSquare, - color: Colors.white, - ) - : const Icon( - EvaIcons.square, - color: Colors.white, - ), - title: Text(device.label), - ), - onTap: () => _selectAudioOutput(device), - ); - }).toList() - ]; - }, - ), + ), + if (_audioOutputs != null) + ..._audioOutputs!.map((device) { + return PopupMenuItem( + value: device, + child: ListTile( + leading: (device.deviceId == + Hardware.instance.selectedAudioOutput?.deviceId) + ? const Icon( + EvaIcons.checkmarkSquare, + color: Colors.white, + ) + : const Icon( + EvaIcons.square, + color: Colors.white, + ), + title: Text(device.label), + ), + onTap: () => _selectAudioOutput(device), + ); + }).toList() + ]; + }, + ), if (participant.isCameraEnabled()) PopupMenuButton( icon: const Icon(EvaIcons.video), diff --git a/example/lib/widgets/participant.dart b/example/lib/widgets/participant.dart index 29dda3c2..070f6e52 100644 --- a/example/lib/widgets/participant.dart +++ b/example/lib/widgets/participant.dart @@ -82,20 +82,12 @@ abstract class _ParticipantWidgetState VideoTrack? get activeVideoTrack; TrackPublication? get videoPublication; TrackPublication? get firstAudioPublication; - List? _audioOutputs; - MediaDevice? _selectedAudioDevice; @override void initState() { super.initState(); widget.participant.addListener(_onParticipantChanged); _onParticipantChanged(); - Hardware.instance.audioOutputs().then((value) { - setState(() { - _audioOutputs = value; - _selectedAudioDevice = _audioOutputs?.firstOrNull; - }); - }); } @override @@ -116,14 +108,6 @@ abstract class _ParticipantWidgetState // since the updated values are computed properties. void _onParticipantChanged() => setState(() {}); - void _onSelectAudioOutput(MediaDevice device) { - var audioTrack = firstAudioPublication?.track as RemoteAudioTrack; - audioTrack.setAudioOutput(device.deviceId); - setState(() { - _selectedAudioDevice = device; - }); - } - // Widgets to show above the info bar List extraWidgets(bool isScreenShare) => []; @@ -146,8 +130,10 @@ abstract class _ParticipantWidgetState InkWell( onTap: () => setState(() => _visible = !_visible), child: activeVideoTrack != null && !activeVideoTrack!.muted - ? VideoTrackRenderer(activeVideoTrack!, - fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain) + ? VideoTrackRenderer( + activeVideoTrack!, + fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain, + ) : const NoVideoWidget(), ), @@ -158,9 +144,7 @@ abstract class _ParticipantWidgetState crossAxisAlignment: CrossAxisAlignment.stretch, mainAxisSize: MainAxisSize.min, children: [ - ...extraWidgets( - widget.isScreenShare, - ), + ...extraWidgets(widget.isScreenShare), ParticipantInfoWidget( title: widget.participant.name.isNotEmpty ? '${widget.participant.name} (${widget.participant.identity})' @@ -227,13 +211,6 @@ class _RemoteParticipantWidgetState pub: firstAudioPublication!, icon: EvaIcons.volumeUp, ), - if (lkPlatformIs(PlatformType.web)) - RemoteTrackAudioOutputSelectMenuWidget( - audioOutputs: _audioOutputs ?? [], - selected: _selectedAudioDevice, - onSelected: _onSelectAudioOutput, - icon: EvaIcons.speaker, - ), ], ), ]; @@ -276,40 +253,3 @@ class RemoteTrackPublicationMenuWidget extends StatelessWidget { ), ); } - -class RemoteTrackAudioOutputSelectMenuWidget extends StatelessWidget { - final IconData icon; - final List audioOutputs; - final MediaDevice? selected; - final Function(MediaDevice) onSelected; - const RemoteTrackAudioOutputSelectMenuWidget({ - required this.audioOutputs, - required this.onSelected, - required this.selected, - required this.icon, - Key? key, - }) : super(key: key); - - @override - Widget build(BuildContext context) => Material( - color: Colors.black.withOpacity(0.3), - child: PopupMenuButton( - tooltip: 'Select AudioOutput', - icon: Icon(icon), - onSelected: (value) => value(), - itemBuilder: (BuildContext context) => >[ - ...audioOutputs - .map((e) => PopupMenuItem( - child: ListTile( - trailing: (e.deviceId == selected?.deviceId) - ? const Icon(Icons.check, color: Colors.white) - : null, - title: Text(e.label), - ), - value: () => onSelected(e), - )) - .toList(), - ], - ), - ); -} From b7b2bf1e677ce395d62bed4aaa317988d8909ae4 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 20:53:06 +0800 Subject: [PATCH 04/11] add RoomHardwareManagementMethods to Room. --- example/lib/widgets/controls.dart | 40 +++++------- lib/src/core/room.dart | 99 ++++++++++++++++++++++++++++- lib/src/hardware/hardware.dart | 32 ++-------- lib/src/options.dart | 34 ++++++++++ lib/src/participant/local.dart | 7 +- lib/src/participant/remote.dart | 9 +++ lib/src/track/audio_management.dart | 6 -- lib/src/track/local/audio.dart | 10 +++ lib/src/track/local/local.dart | 5 -- lib/src/track/options.dart | 37 +++++++++++ 10 files changed, 215 insertions(+), 64 deletions(-) diff --git a/example/lib/widgets/controls.dart b/example/lib/widgets/controls.dart index 5c2256c6..3856a75b 100644 --- a/example/lib/widgets/controls.dart +++ b/example/lib/widgets/controls.dart @@ -33,7 +33,6 @@ class _ControlsWidgetState extends State { List? _audioInputs; List? _audioOutputs; List? _videoInputs; - MediaDevice? _selectedVideoInput; StreamSubscription? _subscription; @@ -61,7 +60,6 @@ class _ControlsWidgetState extends State { _audioInputs = devices.where((d) => d.kind == 'audioinput').toList(); _audioOutputs = devices.where((d) => d.kind == 'audiooutput').toList(); _videoInputs = devices.where((d) => d.kind == 'videoinput').toList(); - _selectedVideoInput = _videoInputs?.first; setState(() {}); } @@ -94,23 +92,18 @@ class _ControlsWidgetState extends State { } void _selectAudioOutput(MediaDevice device) async { - await Hardware.instance.selectAudioOutput(device); + await widget.room.setAudioOutputDevice(device); setState(() {}); } void _selectAudioInput(MediaDevice device) async { - await Hardware.instance.selectAudioInput(device); + await widget.room.setAudioInputDevice(device); setState(() {}); } void _selectVideoInput(MediaDevice device) async { - final track = participant.videoTracks.firstOrNull?.track; - if (track == null) return; - if (_selectedVideoInput?.deviceId != device.deviceId) { - await track.switchCamera(device.deviceId); - _selectedVideoInput = device; - setState(() {}); - } + await widget.room.setVideoInputDevice(device); + setState(() {}); } void _toggleCamera() async { @@ -293,8 +286,7 @@ class _ControlsWidgetState extends State { value: device, child: ListTile( leading: (device.deviceId == - Hardware - .instance.selectedAudioInput?.deviceId) + widget.room.selectedAudioInputDeviceId) ? const Icon( EvaIcons.checkmarkSquare, color: Colors.white, @@ -337,7 +329,7 @@ class _ControlsWidgetState extends State { value: device, child: ListTile( leading: (device.deviceId == - Hardware.instance.selectedAudioOutput?.deviceId) + widget.room.selectedAudioOutputDeviceId) ? const Icon( EvaIcons.checkmarkSquare, color: Colors.white, @@ -375,16 +367,16 @@ class _ControlsWidgetState extends State { return PopupMenuItem( value: device, child: ListTile( - leading: - (device.deviceId == _selectedVideoInput?.deviceId) - ? const Icon( - EvaIcons.checkmarkSquare, - color: Colors.white, - ) - : const Icon( - EvaIcons.square, - color: Colors.white, - ), + leading: (device.deviceId == + widget.room.selectedVideoInputDeviceId) + ? const Icon( + EvaIcons.checkmarkSquare, + color: Colors.white, + ) + : const Icon( + EvaIcons.square, + color: Colors.white, + ), title: Text(device.label), ), onTap: () => _selectVideoInput(device), diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index 485804b3..739f5d63 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; +import 'package:livekit_client/src/hardware/hardware.dart'; import 'package:livekit_client/src/support/app_state.dart'; import 'package:meta/meta.dart'; @@ -41,7 +42,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { ConnectOptions get connectOptions => engine.connectOptions; RoomOptions get roomOptions => engine.roomOptions; - /// map of SID to RemoteParticipant + //of SID to RemoteParticipant UnmodifiableMapView get participants => UnmodifiableMapView(_participants); final _participants = {}; @@ -343,6 +344,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { event.stream, trackSid, receiver: event.receiver, + audioOutputOptions: roomOptions.defaultAudioOutputOptions, ); } on TrackSubscriptionExceptionEvent catch (event) { logger.severe('addSubscribedMediaTrack() throwed ${event}'); @@ -640,3 +642,98 @@ extension RoomDebugMethods on Room { switchCandidate: switchCandidate); } } + +/// Room extension methods for managing audio, video. +extension RoomHardwareManagementMethods on Room { + /// Get current audio output device. + String? get selectedAudioOutputDeviceId => + roomOptions.defaultAudioOutputOptions.deviceId; + + /// Get current audio input device. + String? get selectedAudioInputDeviceId => + roomOptions.defaultAudioCaptureOptions.deviceId; + + /// Get current video input device. + String? get selectedVideoInputDeviceId => + roomOptions.defaultCameraCaptureOptions.deviceId; + + /// Get mobile device's speaker status. + bool? get speakerOn => roomOptions.defaultAudioOutputOptions.speakerOn; + + /// Set audio output device. + Future setAudioOutputDevice(MediaDevice device) async { + if (lkPlatformIs(PlatformType.web)) { + participants.forEach((_, participant) { + for (var audioTrack in participant.audioTracks) { + audioTrack.track?.setAudioOutput(device.deviceId); + } + }); + Hardware.instance.selectedAudioOutput = device; + } else { + await Hardware.instance.selectAudioOutput(device); + } + engine.roomOptions = engine.roomOptions.copyWith( + defaultAudioOutputOptions: roomOptions.defaultAudioOutputOptions.copyWith( + deviceId: device.deviceId, + ), + ); + } + + /// Set audio input device. + Future setAudioInputDevice(MediaDevice device) async { + if (lkPlatformIs(PlatformType.web) && localParticipant != null) { + for (var audioTrack in localParticipant!.audioTracks) { + await audioTrack.track?.setDeviceId(device.deviceId); + } + Hardware.instance.selectedAudioInput = device; + } else { + await Hardware.instance.selectAudioInput(device); + } + engine.roomOptions = engine.roomOptions.copyWith( + defaultAudioCaptureOptions: + roomOptions.defaultAudioCaptureOptions.copyWith( + deviceId: device.deviceId, + ), + ); + } + + /// Set video input device. + Future setVideoInputDevice(MediaDevice device) async { + final track = localParticipant?.videoTracks.firstOrNull?.track; + if (track == null) return; + if (selectedVideoInputDeviceId != device.deviceId) { + await track.switchCamera(device.deviceId); + Hardware.instance.selectedVideoInput = device; + } + engine.roomOptions = engine.roomOptions.copyWith( + defaultCameraCaptureOptions: + roomOptions.defaultCameraCaptureOptions.copyWith( + deviceId: device.deviceId, + ), + ); + } + + Future setSpeakerOn(bool speakerOn) async { + if (lkPlatformIs(PlatformType.iOS) || lkPlatformIs(PlatformType.android)) { + await Hardware.instance.setSpeakerphoneOn(speakerOn); + engine.roomOptions = engine.roomOptions.copyWith( + defaultAudioOutputOptions: + roomOptions.defaultAudioOutputOptions.copyWith( + speakerOn: speakerOn, + ), + ); + } + } + + /// Apply audio output device settings. + @internal + Future applyAudioSpeakerSettings() async { + if (roomOptions.defaultAudioOutputOptions.speakerOn != null) { + if (lkPlatformIs(PlatformType.iOS) || + lkPlatformIs(PlatformType.android)) { + await Hardware.instance.setSpeakerphoneOn( + roomOptions.defaultAudioOutputOptions.speakerOn!); + } + } + } +} diff --git a/lib/src/hardware/hardware.dart b/lib/src/hardware/hardware.dart index 8d2fedf9..2253a687 100644 --- a/lib/src/hardware/hardware.dart +++ b/lib/src/hardware/hardware.dart @@ -2,7 +2,8 @@ import 'dart:async'; import 'package:collection/collection.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -import 'package:meta/meta.dart'; + +import '../logger.dart'; class MediaDevice { const MediaDevice(this.deviceId, this.label, this.kind); @@ -41,7 +42,6 @@ class Hardware { devices.firstWhereOrNull((element) => element.kind == 'audiooutput'); selectedVideoInput ??= devices.firstWhereOrNull((element) => element.kind == 'videoinput'); - _lastDevices = devices; speakerOn = true; }); } @@ -59,8 +59,6 @@ class Hardware { bool? speakerOn; - List? _lastDevices; - Future> enumerateDevices({String? type}) async { var infos = await rtc.navigator.mediaDevices.enumerateDevices(); var devices = @@ -85,7 +83,8 @@ class Hardware { Future selectAudioOutput(MediaDevice device) async { if (rtc.WebRTC.platformIsWeb) { - throw UnimplementedError('selectAudioOutput not support on web'); + logger.warning('selectAudioOutput not support on web'); + return; } selectedAudioOutput = device; await rtc.Helper.selectAudioOutput(device.deviceId); @@ -93,8 +92,9 @@ class Hardware { Future selectAudioInput(MediaDevice device) async { if (rtc.WebRTC.platformIsWeb) { - throw UnimplementedError( + logger.warning( 'selectAudioInput is only supported on Android/Windows/macOS'); + return; } selectedAudioInput = device; await rtc.Helper.selectAudioInput(device.deviceId); @@ -105,7 +105,7 @@ class Hardware { speakerOn = enable; await rtc.Helper.setSpeakerphoneOn(enable); } else { - throw UnimplementedError('setSpeakerphoneOn only support on iOS/Android'); + logger.warning('setSpeakerphoneOn only support on iOS/Android'); } } @@ -139,23 +139,5 @@ class Hardware { selectedVideoInput ??= devices.firstWhereOrNull((element) => element.kind == 'videoinput'); onDeviceChange.add(devices); - _lastDevices = devices; - } - - @internal - Future applyAudioSettings() async { - var devices = await enumerateDevices(); - // if devices no changes, reselect audio input/output - if (_lastDevices?.equals(devices) == true) { - if (selectedAudioInput != null && !rtc.WebRTC.platformIsWeb) { - await selectAudioInput(selectedAudioInput!); - } - if (selectedAudioOutput != null && !rtc.WebRTC.platformIsWeb) { - await selectAudioOutput(selectedAudioOutput!); - } - } - if (speakerOn != null && rtc.WebRTC.platformIsMobile) { - await setSpeakerphoneOn(speakerOn!); - } } } diff --git a/lib/src/options.dart b/lib/src/options.dart index 07681d20..561e83ce 100644 --- a/lib/src/options.dart +++ b/lib/src/options.dart @@ -67,6 +67,8 @@ class RoomOptions { /// Default options used when publishing a [LocalAudioTrack]. final AudioPublishOptions defaultAudioPublishOptions; + final AudioOutputOptions defaultAudioOutputOptions; + /// AdaptiveStream lets LiveKit automatically manage quality of subscribed /// video tracks to optimize for bandwidth and CPU. /// When attached video elements are visible, it'll choose an appropriate @@ -92,10 +94,42 @@ class RoomOptions { this.defaultAudioCaptureOptions = const AudioCaptureOptions(), this.defaultVideoPublishOptions = const VideoPublishOptions(), this.defaultAudioPublishOptions = const AudioPublishOptions(), + this.defaultAudioOutputOptions = const AudioOutputOptions(), this.adaptiveStream = false, this.dynacast = false, this.stopLocalTrackOnUnpublish = true, }); + + RoomOptions copyWith({ + CameraCaptureOptions? defaultCameraCaptureOptions, + ScreenShareCaptureOptions? defaultScreenShareCaptureOptions, + AudioCaptureOptions? defaultAudioCaptureOptions, + VideoPublishOptions? defaultVideoPublishOptions, + AudioPublishOptions? defaultAudioPublishOptions, + AudioOutputOptions? defaultAudioOutputOptions, + bool? adaptiveStream, + bool? dynacast, + bool? stopLocalTrackOnUnpublish, + }) { + return RoomOptions( + defaultCameraCaptureOptions: + defaultCameraCaptureOptions ?? this.defaultCameraCaptureOptions, + defaultScreenShareCaptureOptions: defaultScreenShareCaptureOptions ?? + this.defaultScreenShareCaptureOptions, + defaultAudioCaptureOptions: + defaultAudioCaptureOptions ?? this.defaultAudioCaptureOptions, + defaultVideoPublishOptions: + defaultVideoPublishOptions ?? this.defaultVideoPublishOptions, + defaultAudioPublishOptions: + defaultAudioPublishOptions ?? this.defaultAudioPublishOptions, + defaultAudioOutputOptions: + defaultAudioOutputOptions ?? this.defaultAudioOutputOptions, + adaptiveStream: adaptiveStream ?? this.adaptiveStream, + dynacast: dynacast ?? this.dynacast, + stopLocalTrackOnUnpublish: + stopLocalTrackOnUnpublish ?? this.stopLocalTrackOnUnpublish, + ); + } } /// Options used when publishing video. diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index 9b10e025..a0558527 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -7,6 +7,7 @@ import '../core/signal_client.dart'; import '../events.dart'; import '../exceptions.dart'; import '../extensions.dart'; +import '../hardware/hardware.dart'; import '../logger.dart'; import '../options.dart'; import '../proto/livekit_models.pb.dart' as lk_models; @@ -88,7 +89,7 @@ class LocalParticipant extends Participant { // did publish await track.onPublish(); - await track.applyAudioSettings(); + await room.applyAudioSpeakerSettings(); [events, room.events].emit(LocalTrackPublishedEvent( participant: this, @@ -229,7 +230,7 @@ class LocalParticipant extends Participant { // did unpublish await track.onUnpublish(); - await track.applyAudioSettings(); + await room.applyAudioSpeakerSettings(); } if (notify) { @@ -321,7 +322,7 @@ class LocalParticipant extends Participant { await publication.mute(); } } - await publication.track?.applyAudioSettings(); + await room.applyAudioSpeakerSettings(); return publication; } else if (enabled) { if (source == TrackSource.camera) { diff --git a/lib/src/participant/remote.dart b/lib/src/participant/remote.dart index cfc2e5ff..f31a0b9e 100644 --- a/lib/src/participant/remote.dart +++ b/lib/src/participant/remote.dart @@ -1,4 +1,5 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:livekit_client/livekit_client.dart'; import 'package:meta/meta.dart'; import '../core/room.dart'; @@ -75,6 +76,7 @@ class RemoteParticipant extends Participant { rtc.MediaStream stream, String trackSid, { rtc.RTCRtpReceiver? receiver, + AudioOutputOptions audioOutputOptions = const AudioOutputOptions(), }) async { logger.fine('addSubscribedMediaTrack()'); @@ -123,6 +125,13 @@ class RemoteParticipant extends Participant { } await track.start(); + if (pub.kind == lk_models.TrackType.AUDIO) { + if (audioOutputOptions.deviceId != null) { + await (track as RemoteAudioTrack) + .setAudioOutput(audioOutputOptions.deviceId!); + } + } + await pub.updateTrack(track); await pub.updateSubscriptionAllowed(true); addTrackPublication(pub); diff --git a/lib/src/track/audio_management.dart b/lib/src/track/audio_management.dart index ff512a96..aed8fd7b 100644 --- a/lib/src/track/audio_management.dart +++ b/lib/src/track/audio_management.dart @@ -28,12 +28,6 @@ int _localTrackCount = 0; int _remoteTrackCount = 0; mixin LocalAudioManagementMixin on LocalTrack, AudioTrack { - @override - Future applyAudioSettings() async { - var hardware = Hardware.instance; - await hardware.applyAudioSettings(); - } - @override Future onPublish() async { final didUpdate = await super.onPublish(); diff --git a/lib/src/track/local/audio.dart b/lib/src/track/local/audio.dart index ebe21167..fce84022 100644 --- a/lib/src/track/local/audio.dart +++ b/lib/src/track/local/audio.dart @@ -15,6 +15,16 @@ class LocalAudioTrack extends LocalTrack @override covariant AudioCaptureOptions currentOptions; + Future setDeviceId(String deviceId) async { + if (currentOptions.deviceId == deviceId) { + return; + } + currentOptions = currentOptions.copyWith(deviceId: deviceId); + if (!muted) { + await restartTrack(); + } + } + // private constructor @internal LocalAudioTrack( diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index e2742bcb..c8dceb74 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -183,11 +183,6 @@ abstract class LocalTrack extends Track { )); } - @internal - Future applyAudioSettings() async { - logger.fine('$objectId.applyAudioSettings()'); - } - @internal @mustCallSuper Future onPublish() async { diff --git a/lib/src/track/options.dart b/lib/src/track/options.dart index 8d3d1ac9..873f609e 100644 --- a/lib/src/track/options.dart +++ b/lib/src/track/options.dart @@ -1,6 +1,7 @@ import 'package:flutter/foundation.dart' show kIsWeb; import 'package:flutter_webrtc/flutter_webrtc.dart'; +import '../support/platform.dart'; import '../track/local/audio.dart'; import '../track/local/video.dart'; import '../types/video_parameters.dart'; @@ -229,4 +230,40 @@ class AudioCaptureOptions extends LocalTrackOptions { } return constraints; } + + AudioCaptureOptions copyWith({ + String? deviceId, + bool? noiseSuppression, + bool? echoCancellation, + bool? autoGainControl, + bool? highPassFilter, + bool? typingNoiseDetection, + }) { + return AudioCaptureOptions( + deviceId: deviceId ?? this.deviceId, + noiseSuppression: noiseSuppression ?? this.noiseSuppression, + echoCancellation: echoCancellation ?? this.echoCancellation, + autoGainControl: autoGainControl ?? this.autoGainControl, + highPassFilter: highPassFilter ?? this.highPassFilter, + typingNoiseDetection: typingNoiseDetection ?? this.typingNoiseDetection, + ); + } +} + +class AudioOutputOptions { + /// The deviceId of the output device to use. + final String? deviceId; + + /// If true, the audio will be played on the speaker. + /// for mobile only + final bool? speakerOn; + + const AudioOutputOptions({this.deviceId, this.speakerOn}); + + AudioOutputOptions copyWith({String? deviceId, bool? speakerOn}) { + return AudioOutputOptions( + deviceId: deviceId ?? this.deviceId, + speakerOn: speakerOn ?? this.speakerOn, + ); + } } From 008adc3ce054cd2506184709827c7fe8dd09e4ad Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 21:03:32 +0800 Subject: [PATCH 05/11] update. --- lib/src/core/room.dart | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index 739f5d63..085cb7f0 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -647,15 +647,18 @@ extension RoomDebugMethods on Room { extension RoomHardwareManagementMethods on Room { /// Get current audio output device. String? get selectedAudioOutputDeviceId => - roomOptions.defaultAudioOutputOptions.deviceId; + roomOptions.defaultAudioOutputOptions.deviceId ?? + Hardware.instance.selectedAudioOutput?.deviceId; /// Get current audio input device. String? get selectedAudioInputDeviceId => - roomOptions.defaultAudioCaptureOptions.deviceId; + roomOptions.defaultAudioCaptureOptions.deviceId ?? + Hardware.instance.selectedAudioInput?.deviceId; /// Get current video input device. String? get selectedVideoInputDeviceId => - roomOptions.defaultCameraCaptureOptions.deviceId; + roomOptions.defaultCameraCaptureOptions.deviceId ?? + Hardware.instance.selectedVideoInput?.deviceId; /// Get mobile device's speaker status. bool? get speakerOn => roomOptions.defaultAudioOutputOptions.speakerOn; From 868a2805357a74cd0797e2c137e8f352f99ce835 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 21:09:31 +0800 Subject: [PATCH 06/11] fix analyze. --- lib/src/participant/local.dart | 1 - lib/src/participant/remote.dart | 2 +- lib/src/track/audio_management.dart | 1 - lib/src/track/options.dart | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index a0558527..acdc76f8 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -7,7 +7,6 @@ import '../core/signal_client.dart'; import '../events.dart'; import '../exceptions.dart'; import '../extensions.dart'; -import '../hardware/hardware.dart'; import '../logger.dart'; import '../options.dart'; import '../proto/livekit_models.pb.dart' as lk_models; diff --git a/lib/src/participant/remote.dart b/lib/src/participant/remote.dart index f31a0b9e..99b8d5b6 100644 --- a/lib/src/participant/remote.dart +++ b/lib/src/participant/remote.dart @@ -1,5 +1,4 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -import 'package:livekit_client/livekit_client.dart'; import 'package:meta/meta.dart'; import '../core/room.dart'; @@ -9,6 +8,7 @@ import '../extensions.dart'; import '../logger.dart'; import '../proto/livekit_models.pb.dart' as lk_models; import '../publication/remote.dart'; +import '../track/options.dart'; import '../track/remote/audio.dart'; import '../track/remote/remote.dart'; import '../track/remote/video.dart'; diff --git a/lib/src/track/audio_management.dart b/lib/src/track/audio_management.dart index aed8fd7b..f303e374 100644 --- a/lib/src/track/audio_management.dart +++ b/lib/src/track/audio_management.dart @@ -1,4 +1,3 @@ -import 'package:livekit_client/src/hardware/hardware.dart'; import 'package:synchronized/synchronized.dart' as sync; import '../logger.dart'; diff --git a/lib/src/track/options.dart b/lib/src/track/options.dart index 873f609e..8a78cea1 100644 --- a/lib/src/track/options.dart +++ b/lib/src/track/options.dart @@ -1,7 +1,6 @@ import 'package:flutter/foundation.dart' show kIsWeb; import 'package:flutter_webrtc/flutter_webrtc.dart'; -import '../support/platform.dart'; import '../track/local/audio.dart'; import '../track/local/video.dart'; import '../types/video_parameters.dart'; From fcff56ca3f74bb577e49b66554f9955a2fb425b8 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 21:11:10 +0800 Subject: [PATCH 07/11] update. --- lib/src/core/room.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index 085cb7f0..ce8d9f05 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -42,7 +42,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { ConnectOptions get connectOptions => engine.connectOptions; RoomOptions get roomOptions => engine.roomOptions; - //of SID to RemoteParticipant + /// map of SID to RemoteParticipant UnmodifiableMapView get participants => UnmodifiableMapView(_participants); final _participants = {}; From 3e25bd84d9890f2d84001e6c5bf37b763568afb7 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 21:19:42 +0800 Subject: [PATCH 08/11] update. --- lib/src/participant/remote.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/src/participant/remote.dart b/lib/src/participant/remote.dart index 99b8d5b6..daec9a46 100644 --- a/lib/src/participant/remote.dart +++ b/lib/src/participant/remote.dart @@ -8,6 +8,7 @@ import '../extensions.dart'; import '../logger.dart'; import '../proto/livekit_models.pb.dart' as lk_models; import '../publication/remote.dart'; +import '../support/platform.dart'; import '../track/options.dart'; import '../track/remote/audio.dart'; import '../track/remote/remote.dart'; @@ -125,7 +126,10 @@ class RemoteParticipant extends Participant { } await track.start(); - if (pub.kind == lk_models.TrackType.AUDIO) { + + /// Apply audio output selection for the web. + if (pub.kind == lk_models.TrackType.AUDIO && + lkPlatformIs(PlatformType.web)) { if (audioOutputOptions.deviceId != null) { await (track as RemoteAudioTrack) .setAudioOutput(audioOutputOptions.deviceId!); From e54e69ead71ce7578de07101376936e263c316b0 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 21:28:58 +0800 Subject: [PATCH 09/11] chore: Reset sinkId for web when remoteTrack replays. --- lib/src/track/remote/audio.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/src/track/remote/audio.dart b/lib/src/track/remote/audio.dart index d522a8ef..983e77df 100644 --- a/lib/src/track/remote/audio.dart +++ b/lib/src/track/remote/audio.dart @@ -11,7 +11,7 @@ import '../web/_audio_api.dart' if (dart.library.html) '../web/_audio_html.dart' class RemoteAudioTrack extends RemoteTrack with AudioTrack, RemoteAudioManagementMixin { - // + String? _deviceId; RemoteAudioTrack(String name, TrackSource source, rtc.MediaStream stream, rtc.MediaStreamTrack track, {rtc.RTCRtpReceiver? receiver}) @@ -30,6 +30,9 @@ class RemoteAudioTrack extends RemoteTrack if (didStart) { // web support audio.startAudio(getCid(), mediaStreamTrack); + if (_deviceId != null) { + audio.setAudioOutput(getCid(), _deviceId!); + } } return didStart; } @@ -46,5 +49,6 @@ class RemoteAudioTrack extends RemoteTrack Future setAudioOutput(String deviceId) async { audio.setAudioOutput(getCid(), deviceId); + _deviceId = deviceId; } } From fd252d6aaafa1a8eeb1893863df4640d5e60e086 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 21:35:47 +0800 Subject: [PATCH 10/11] rename setAudioOutput to setSinkId for flutter web. --- lib/src/core/room.dart | 2 +- lib/src/participant/remote.dart | 2 +- lib/src/track/remote/audio.dart | 6 +++--- lib/src/track/web/_audio_api.dart | 2 +- lib/src/track/web/_audio_html.dart | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index ce8d9f05..f35dcaed 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -668,7 +668,7 @@ extension RoomHardwareManagementMethods on Room { if (lkPlatformIs(PlatformType.web)) { participants.forEach((_, participant) { for (var audioTrack in participant.audioTracks) { - audioTrack.track?.setAudioOutput(device.deviceId); + audioTrack.track?.setSinkId(device.deviceId); } }); Hardware.instance.selectedAudioOutput = device; diff --git a/lib/src/participant/remote.dart b/lib/src/participant/remote.dart index daec9a46..4d177b43 100644 --- a/lib/src/participant/remote.dart +++ b/lib/src/participant/remote.dart @@ -132,7 +132,7 @@ class RemoteParticipant extends Participant { lkPlatformIs(PlatformType.web)) { if (audioOutputOptions.deviceId != null) { await (track as RemoteAudioTrack) - .setAudioOutput(audioOutputOptions.deviceId!); + .setSinkId(audioOutputOptions.deviceId!); } } diff --git a/lib/src/track/remote/audio.dart b/lib/src/track/remote/audio.dart index 983e77df..ff679405 100644 --- a/lib/src/track/remote/audio.dart +++ b/lib/src/track/remote/audio.dart @@ -31,7 +31,7 @@ class RemoteAudioTrack extends RemoteTrack // web support audio.startAudio(getCid(), mediaStreamTrack); if (_deviceId != null) { - audio.setAudioOutput(getCid(), _deviceId!); + audio.setSinkId(getCid(), _deviceId!); } } return didStart; @@ -47,8 +47,8 @@ class RemoteAudioTrack extends RemoteTrack return didStop; } - Future setAudioOutput(String deviceId) async { - audio.setAudioOutput(getCid(), deviceId); + Future setSinkId(String deviceId) async { + audio.setSinkId(getCid(), deviceId); _deviceId = deviceId; } } diff --git a/lib/src/track/web/_audio_api.dart b/lib/src/track/web/_audio_api.dart index 961726a5..b9c215ae 100644 --- a/lib/src/track/web/_audio_api.dart +++ b/lib/src/track/web/_audio_api.dart @@ -8,6 +8,6 @@ void stopAudio(String id) { // do nothing } -void setAudioOutput(String id, String deviceId) { +void setSinkId(String id, String deviceId) { // do nothing } diff --git a/lib/src/track/web/_audio_html.dart b/lib/src/track/web/_audio_html.dart index 000e2307..92c3cf74 100644 --- a/lib/src/track/web/_audio_html.dart +++ b/lib/src/track/web/_audio_html.dart @@ -52,7 +52,7 @@ html.DivElement findOrCreateAudioContainer() { return div as html.DivElement; } -void setAudioOutput(String id, String deviceId) { +void setSinkId(String id, String deviceId) { final audioElement = html.document.getElementById(audioPrefix + id); if (audioElement is html.AudioElement) { audioElement.setSinkId(deviceId); From de26bc73eb70f733dabd54fa83b30dcf8e486798 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 6 Mar 2023 22:17:38 +0800 Subject: [PATCH 11/11] chore: Make sure AudioElement has `setSinkId` method. --- lib/src/track/web/_audio_html.dart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/src/track/web/_audio_html.dart b/lib/src/track/web/_audio_html.dart index 92c3cf74..09a73ad9 100644 --- a/lib/src/track/web/_audio_html.dart +++ b/lib/src/track/web/_audio_html.dart @@ -1,5 +1,5 @@ import 'dart:html' as html; - +import 'dart:js_util' as jsutil; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; // ignore: implementation_imports @@ -54,7 +54,8 @@ html.DivElement findOrCreateAudioContainer() { void setSinkId(String id, String deviceId) { final audioElement = html.document.getElementById(audioPrefix + id); - if (audioElement is html.AudioElement) { + if (audioElement is html.AudioElement && + jsutil.hasProperty(audioElement, 'setSinkId')) { audioElement.setSinkId(deviceId); } }