Rework set audio output device, configurable peerConnection timeout. (#…

…472)

Previously when `switchActiveDevice` is called, it doesn't remember
that preference for newly attached audio tracks. Reworked setSinkId to
model audioContext handling.

Also fixing a potential timing issue when a new track can be created
without respecting default input deviceId.

A minor cleanup by removing redundant `RemoteTrack` type.

.changeset/lucky-ghosts-tease.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Fixes switchAudioDevice not respecting those preferences for future tracks

.changeset/silver-suns-refuse.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+peer connnection timeout is now configurable

src/options.ts

@@ -1,10 +1,11 @@
+import type { ReconnectPolicy } from './room/ReconnectPolicy';
 import type {
   AudioCaptureOptions,
+  AudioOutputOptions,
   TrackPublishDefaults,
   VideoCaptureOptions,
 } from './room/track/options';
 import type { AdaptiveStreamSettings } from './room/track/types';
-import type { ReconnectPolicy } from './room/ReconnectPolicy';
 
 /**
  * @internal
@@ -43,6 +44,11 @@ export interface InternalRoomOptions {
    */
   publishDefaults?: TrackPublishDefaults;
 
+  /**
+   * audio output for the room
+   */
+  audioOutput?: AudioOutputOptions;
+
   /**
    * should local tracks be stopped when they are unpublished. defaults to true
    * set this to false if you would prefer to clean up unpublished local tracks manually.
@@ -80,6 +86,9 @@ export interface InternalRoomConnectOptions {
   /** autosubscribe to room tracks after joining, defaults to true */
   autoSubscribe: boolean;
 
+  /** amount of time for PeerConnection to be established, defaults to 15s */
+  peerConnectionTimeout: number;
+
   /**
    * use to override any RTCConfiguration options.
    */

src/room/RTCEngine.ts

@@ -20,6 +20,7 @@ import {
   SignalTarget,
   TrackPublishedResponse,
 } from '../proto/livekit_rtc';
+import { roomConnectOptionDefaults } from './defaults';
 import {
   ConnectionError,
   NegotiationError,
@@ -46,7 +47,6 @@ const lossyDataChannel = '_lossy';
 const reliableDataChannel = '_reliable';
 const minReconnectWait = 2 * 1000;
 const leaveReconnect = 'leave-reconnect';
-export const maxICEConnectTimeout = 15 * 1000;
 
 enum PCState {
   New,
@@ -66,6 +66,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
   rtcConfig: RTCConfiguration = {};
 
+  peerConnectionTimeout: number = roomConnectOptionDefaults.peerConnectionTimeout;
+
   get isClosed() {
     return this._isClosed;
   }
@@ -837,7 +839,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     this.pcState = PCState.Reconnecting;
 
     log.debug('waiting for peer connection to reconnect');
-    while (now - startTime < maxICEConnectTimeout) {
+    while (now - startTime < this.peerConnectionTimeout) {
       if (this.primaryPC === undefined) {
         // we can abort early, connection is hosed
         break;
@@ -900,7 +902,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }
 
     // wait until publisher ICE connected
-    const endTime = new Date().getTime() + maxICEConnectTimeout;
+    const endTime = new Date().getTime() + this.peerConnectionTimeout;
     while (new Date().getTime() < endTime) {
       if (this.publisher.isICEConnected && this.dataChannelForKind(kind)?.readyState === 'open') {
         return;

src/room/Room.ts

@@ -40,16 +40,17 @@ import LocalParticipant from './participant/LocalParticipant';
 import type Participant from './participant/Participant';
 import type { ConnectionQuality } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
-import RTCEngine, { maxICEConnectTimeout } from './RTCEngine';
+import RTCEngine from './RTCEngine';
 import LocalAudioTrack from './track/LocalAudioTrack';
 import type LocalTrackPublication from './track/LocalTrackPublication';
 import LocalVideoTrack from './track/LocalVideoTrack';
+import type RemoteTrack from './track/RemoteTrack';
 import RemoteTrackPublication from './track/RemoteTrackPublication';
 import { Track } from './track/Track';
 import type { TrackPublication } from './track/TrackPublication';
-import type { AdaptiveStreamSettings, RemoteTrack } from './track/types';
+import type { AdaptiveStreamSettings } from './track/types';
 import { getNewAudioContext } from './track/utils';
-import { Future, isWeb, unpackStreamId } from './utils';
+import { Future, isWeb, supportsSetSinkId, unpackStreamId } from './utils';
 
 export enum ConnectionState {
   Disconnected = 'disconnected',
@@ -260,6 +261,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       if (this.connOptions.rtcConfig) {
         this.engine.rtcConfig = this.connOptions.rtcConfig;
       }
+      if (this.connOptions.peerConnectionTimeout) {
+        this.engine.peerConnectionTimeout = this.connOptions.peerConnectionTimeout;
+      }
 
       try {
         const joinResponse = await this.engine.join(
@@ -347,7 +351,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         this.recreateEngine();
         this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
         reject(new ConnectionError('could not connect PeerConnection after timeout'));
-      }, maxICEConnectTimeout);
+      }, this.connOptions.peerConnectionTimeout);
       const abortHandler = () => {
         log.warn('closing engine');
         clearTimeout(connectTimeout);
@@ -550,37 +554,51 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    */
   async switchActiveDevice(kind: MediaDeviceKind, deviceId: string) {
     if (kind === 'audioinput') {
+      const prevDeviceId = this.options.audioCaptureDefaults!.deviceId;
+      this.options.audioCaptureDefaults!.deviceId = deviceId;
       const tracks = Array.from(this.localParticipant.audioTracks.values()).filter(
         (track) => track.source === Track.Source.Microphone,
       );
-      await Promise.all(tracks.map((t) => t.audioTrack?.setDeviceId(deviceId)));
-      this.options.audioCaptureDefaults!.deviceId = deviceId;
+      try {
+        await Promise.all(tracks.map((t) => t.audioTrack?.setDeviceId(deviceId)));
+      } catch (e) {
+        this.options.audioCaptureDefaults!.deviceId = prevDeviceId;
+        throw e;
+      }
     } else if (kind === 'videoinput') {
+      const prevDeviceId = this.options.videoCaptureDefaults!.deviceId;
+      this.options.videoCaptureDefaults!.deviceId = deviceId;
       const tracks = Array.from(this.localParticipant.videoTracks.values()).filter(
         (track) => track.source === Track.Source.Camera,
       );
-      await Promise.all(tracks.map((t) => t.videoTrack?.setDeviceId(deviceId)));
-      this.options.videoCaptureDefaults!.deviceId = deviceId;
+      try {
+        await Promise.all(tracks.map((t) => t.videoTrack?.setDeviceId(deviceId)));
+      } catch (e) {
+        this.options.videoCaptureDefaults!.deviceId = prevDeviceId;
+        throw e;
+      }
     } else if (kind === 'audiooutput') {
-      const elements: HTMLMediaElement[] = [];
+      // TODO add support for webaudio mix once the API becomes available https://github.com/WebAudio/web-audio-api/pull/2498
+      if (!supportsSetSinkId()) {
+        throw new Error('cannot switch audio output, setSinkId not supported');
+      }
+      this.options.audioOutput ??= {};
+      const prevDeviceId = this.options.audioOutput.deviceId;
+      this.options.audioOutput.deviceId = deviceId;
+      const promises: Promise<void>[] = [];
       this.participants.forEach((p) => {
-        p.audioTracks.forEach((t) => {
-          if (t.isSubscribed && t.track) {
-            t.track.attachedElements.forEach((e) => {
-              elements.push(e);
-            });
-          }
-        });
+        promises.push(
+          p.setAudioOutput({
+            deviceId,
+          }),
+        );
       });
-
-      await Promise.all(
-        elements.map(async (e) => {
-          if ('setSinkId' in e) {
-            /* @ts-ignore */
-            await e.setSinkId(deviceId);
-          }
-        }),
-      );
+      try {
+        await Promise.all(promises);
+      } catch (e) {
+        this.options.audioOutput.deviceId = prevDeviceId;
+        throw e;
+      }
     }
   }
 

src/room/defaults.ts

@@ -39,4 +39,5 @@ export const roomOptionDefaults: InternalRoomOptions = {
 
 export const roomConnectOptionDefaults: InternalRoomConnectOptions = {
   autoSubscribe: true,
+  peerConnectionTimeout: 15_000,
 } as const;

src/room/participant/Participant.ts

@@ -1,5 +1,6 @@
 import { EventEmitter } from 'events';
 import type TypedEmitter from 'typed-emitter';
+import log from '../../logger';
 import {
   ConnectionQuality as ProtoQuality,
   DataPacket_Kind,
@@ -8,11 +9,10 @@ import {
 } from '../../proto/livekit_models';
 import { ParticipantEvent, TrackEvent } from '../events';
 import type LocalTrackPublication from '../track/LocalTrackPublication';
+import type RemoteTrack from '../track/RemoteTrack';
 import type RemoteTrackPublication from '../track/RemoteTrackPublication';
 import { Track } from '../track/Track';
 import type { TrackPublication } from '../track/TrackPublication';
-import type { RemoteTrack } from '../track/types';
-import log from '../../logger';
 
 export enum ConnectionQuality {
   Excellent = 'excellent',

src/room/participant/RemoteParticipant.ts

@@ -3,12 +3,14 @@ import log from '../../logger';
 import type { ParticipantInfo } from '../../proto/livekit_models';
 import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
 import { ParticipantEvent, TrackEvent } from '../events';
+import type { AudioOutputOptions } from '../track/options';
 import RemoteAudioTrack from '../track/RemoteAudioTrack';
+import type RemoteTrack from '../track/RemoteTrack';
 import RemoteTrackPublication from '../track/RemoteTrackPublication';
 import RemoteVideoTrack from '../track/RemoteVideoTrack';
 import { Track } from '../track/Track';
 import type { TrackPublication } from '../track/TrackPublication';
-import type { AdaptiveStreamSettings, RemoteTrack } from '../track/types';
+import type { AdaptiveStreamSettings } from '../track/types';
 import Participant, { ParticipantEventCallbacks } from './Participant';
 
 export default class RemoteParticipant extends Participant {
@@ -24,6 +26,8 @@ export default class RemoteParticipant extends Participant {
 
   private audioContext?: AudioContext;
 
+  private audioOutput?: AudioOutputOptions;
+
   /** @internal */
   static fromParticipantInfo(signalClient: SignalClient, pi: ParticipantInfo): RemoteParticipant {
     return new RemoteParticipant(signalClient, pi.sid, pi.identity, pi.name, pi.metadata);
@@ -178,7 +182,7 @@ export default class RemoteParticipant extends Participant {
     if (isVideo) {
       track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStreamSettings);
     } else {
-      track = new RemoteAudioTrack(mediaTrack, sid, receiver, this.audioContext);
+      track = new RemoteAudioTrack(mediaTrack, sid, receiver, this.audioContext, this.audioOutput);
     }
 
     // set track info
@@ -313,11 +317,25 @@ export default class RemoteParticipant extends Participant {
    */
   setAudioContext(ctx: AudioContext | undefined) {
     this.audioContext = ctx;
-    this.tracks.forEach(
+    this.audioTracks.forEach(
       (track) => track.track instanceof RemoteAudioTrack && track.track.setAudioContext(ctx),
     );
   }
 
+  /**
+   * @internal
+   */
+  async setAudioOutput(output: AudioOutputOptions) {
+    this.audioOutput = output;
+    const promises: Promise<void>[] = [];
+    this.audioTracks.forEach((pub) => {
+      if (pub.track instanceof RemoteAudioTrack) {
+        promises.push(pub.track.setSinkId(output.deviceId ?? 'default'));
+      }
+    });
+    await Promise.all(promises);
+  }
+
   /** @internal */
   emit<E extends keyof ParticipantEventCallbacks>(
     event: E,

src/room/track/RemoteAudioTrack.ts

@@ -1,8 +1,10 @@
+import log from '../../logger';
+import { TrackEvent } from '../events';
 import { AudioReceiverStats, computeBitrate } from '../stats';
+import { supportsSetSinkId } from '../utils';
+import type { AudioOutputOptions } from './options';
 import RemoteTrack from './RemoteTrack';
 import { Track } from './Track';
-import log from '../../logger';
-import { TrackEvent } from '../events';
 
 export default class RemoteAudioTrack extends RemoteTrack {
   private prevStats?: AudioReceiverStats;
@@ -17,15 +19,21 @@ export default class RemoteAudioTrack extends RemoteTrack {
 
   private webAudioPluginNodes: AudioNode[];
 
+  private sinkId?: string;
+
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
     receiver?: RTCRtpReceiver,
     audioContext?: AudioContext,
+    audioOutput?: AudioOutputOptions,
   ) {
     super(mediaTrack, sid, Track.Kind.Audio, receiver);
     this.audioContext = audioContext;
     this.webAudioPluginNodes = [];
+    if (audioOutput) {
+      this.sinkId = audioOutput.deviceId;
+    }
   }
 
   /**
@@ -58,6 +66,23 @@ export default class RemoteAudioTrack extends RemoteTrack {
     return highestVolume;
   }
 
+  /**
+   * calls setSinkId on all attached elements, if supported
+   * @param deviceId audio output device
+   */
+  async setSinkId(deviceId: string) {
+    this.sinkId = deviceId;
+    await Promise.all(
+      this.attachedElements.map((elm) => {
+        if (!supportsSetSinkId(elm)) {
+          return;
+        }
+        /* @ts-ignore */
+        return elm.setSinkId(deviceId) as Promise<void>;
+      }),
+    );
+  }
+
   attach(): HTMLMediaElement;
   attach(element: HTMLMediaElement): HTMLMediaElement;
   attach(element?: HTMLMediaElement): HTMLMediaElement {
@@ -70,6 +95,10 @@ export default class RemoteAudioTrack extends RemoteTrack {
     if (this.elementVolume) {
       element.volume = this.elementVolume;
     }
+    if (this.sinkId && supportsSetSinkId(element)) {
+      /* @ts-ignore */
+      element.setSinkId(this.sinkId);
+    }
     if (this.audioContext && needsNewWebAudioConnection) {
       log.debug('using audio context mapping');
       this.connectWebAudio(this.audioContext, element);

src/room/track/RemoteTrackPublication.ts

@@ -2,10 +2,10 @@ import log from '../../logger';
 import { TrackInfo, VideoQuality } from '../../proto/livekit_models';
 import { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
 import { TrackEvent } from '../events';
+import type RemoteTrack from './RemoteTrack';
 import RemoteVideoTrack from './RemoteVideoTrack';
 import type { Track } from './Track';
 import { TrackPublication } from './TrackPublication';
-import type { RemoteTrack } from './types';
 
 export default class RemoteTrackPublication extends TrackPublication {
   track?: RemoteTrack = undefined;

src/room/track/TrackPublication.ts

@@ -7,9 +7,9 @@ import { TrackEvent } from '../events';
 import LocalAudioTrack from './LocalAudioTrack';
 import LocalVideoTrack from './LocalVideoTrack';
 import RemoteAudioTrack from './RemoteAudioTrack';
+import type RemoteTrack from './RemoteTrack';
 import RemoteVideoTrack from './RemoteVideoTrack';
 import { Track } from './Track';
-import type { RemoteTrack } from './types';
 
 export class TrackPublication extends (EventEmitter as new () => TypedEventEmitter<PublicationEventCallbacks>) {
   kind: Track.Kind;

src/room/track/options.ts

@@ -168,6 +168,15 @@ export interface AudioCaptureOptions {
   sampleSize?: ConstrainULong;
 }
 
+export interface AudioOutputOptions {
+  /**
+   * deviceId to output audio
+   *
+   * Only supported on browsers where `setSinkId` is available
+   */
+  deviceId?: string;
+}
+
 export interface VideoResolution {
   width: number;
   height: number;