Skip to content

Commit

Permalink
VDI Support 2.x initial implementation (twilio#158)
Browse files Browse the repository at this point in the history 
* VDI Support initial implementation

* Rename connection -> call
  • Loading branch information
@charliesantos
charliesantos authored Apr 26, 2023
1 parent 6add098 commit 3367771
Show file tree
Hide file tree
Showing 8 changed files with 175 additions and 45 deletions.
33 changes: 26 additions & 7 deletions lib/twilio/audiohelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ class AudioHelper extends EventEmitter {
*/
private _audioContext?: AudioContext;

/**
* The enumerateDevices method to use
*/
private _enumerateDevices: any;

/**
* Whether each sound is enabled.
*/
Expand Down Expand Up @@ -174,9 +179,13 @@ class AudioHelper extends EventEmitter {
this._getUserMedia = getUserMedia;
this._mediaDevices = options.mediaDevices || getMediaDevicesInstance();
this._onActiveInputChanged = onActiveInputChanged;
this._enumerateDevices = typeof options.enumerateDevices === 'function'
? options.enumerateDevices
: this._mediaDevices && this._mediaDevices.enumerateDevices;

const isAudioContextSupported: boolean = !!(options.AudioContext || options.audioContext);
const isEnumerationSupported: boolean = !!(this._mediaDevices && this._mediaDevices.enumerateDevices);
const isEnumerationSupported: boolean = !!this._enumerateDevices;

const isSetSinkSupported: boolean = typeof options.setSinkId === 'function';
this.isOutputSelectionSupported = isEnumerationSupported && isSetSinkSupported;
this.isVolumeSupported = isAudioContextSupported;
Expand Down Expand Up @@ -282,7 +291,7 @@ class AudioHelper extends EventEmitter {
* @private
*/
_unbind(): void {
if (!this._mediaDevices) {
if (!this._mediaDevices || !this._enumerateDevices) {
throw new NotSupportedError('Enumeration is not supported');
}

Expand Down Expand Up @@ -400,7 +409,7 @@ class AudioHelper extends EventEmitter {
* Initialize output device enumeration.
*/
private _initializeEnumeration(): void {
if (!this._mediaDevices) {
if (!this._mediaDevices || !this._enumerateDevices) {
throw new NotSupportedError('Enumeration is not supported');
}

Expand Down Expand Up @@ -526,11 +535,11 @@ class AudioHelper extends EventEmitter {
* Update the available input and output devices
*/
private _updateAvailableDevices = (): Promise<void> => {
if (!this._mediaDevices) {
if (!this._mediaDevices || !this._enumerateDevices) {
return Promise.reject('Enumeration not supported');
}

return this._mediaDevices.enumerateDevices().then((devices: MediaDeviceInfo[]) => {
return this._enumerateDevices().then((devices: MediaDeviceInfo[]) => {
this._updateDevices(devices.filter((d: MediaDeviceInfo) => d.kind === 'audiooutput'),
this.availableOutputDevices,
this._removeLostOutput);
Expand Down Expand Up @@ -618,8 +627,13 @@ class AudioHelper extends EventEmitter {
this._inputVolumeSource.disconnect();
}

this._inputVolumeSource = this._audioContext.createMediaStreamSource(this._inputStream);
this._inputVolumeSource.connect(this._inputVolumeAnalyser);
try {
this._inputVolumeSource = this._audioContext.createMediaStreamSource(this._inputStream);
this._inputVolumeSource.connect(this._inputVolumeAnalyser);
} catch (ex) {
this._log.warn('Unable to update volume source', ex);
delete this._inputVolumeSource;
}
}

/**
Expand Down Expand Up @@ -696,6 +710,11 @@ namespace AudioHelper {
*/
audioContext?: AudioContext;

/**
* Overrides the native MediaDevices.enumerateDevices API.
*/
enumerateDevices?: any;

/**
* A custom MediaDevices instance to use.
*/
Expand Down
19 changes: 19 additions & 0 deletions lib/twilio/call.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,7 @@ class Call extends EventEmitter {
isUnifiedPlan: this._isUnifiedPlanDefault,
maxAverageBitrate: this._options.maxAverageBitrate,
preflight: this._options.preflight,
RTCPeerConnection: this._options.RTCPeerConnection,
});

this.on('volume', (inputVolume: number, outputVolume: number): void => {
Expand All @@ -395,6 +396,11 @@ class Call extends EventEmitter {
this._latestOutputVolume = outputVolume;
});

this._mediaHandler.onaudio = (remoteAudio: typeof Audio) => {
this._log.info('Remote audio created');
this.emit('audio', remoteAudio);
};

this._mediaHandler.onvolume = (inputVolume: number, outputVolume: number,
internalInputVolume: number, internalOutputVolume: number) => {
// (rrowland) These values mock the 0 -> 32767 format used by legacy getStats. We should look into
Expand Down Expand Up @@ -1510,6 +1516,14 @@ namespace Call {
*/
declare function acceptEvent(call: Call): void;

/**
* Emitted after the HTMLAudioElement for the remote audio is created.
* @param remoteAudio - The HTMLAudioElement.
* @example `call.on('audio', handler(remoteAudio))`
* @event
*/
declare function audioEvent(remoteAudio: HTMLAudioElement): void;

/**
* Emitted when the {@link Call} is canceled.
* @example `call.on('cancel', () => { })`
Expand Down Expand Up @@ -1875,6 +1889,11 @@ namespace Call {
*/
rtcConstraints?: MediaStreamConstraints;

/**
* The RTCPeerConnection passed to {@link Device} on setup.
*/
RTCPeerConnection?: any;

/**
* Whether the disconnect sound should be played.
*/
Expand Down
37 changes: 31 additions & 6 deletions lib/twilio/device.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -940,7 +940,7 @@ class Device extends EventEmitter {

const config: Call.Config = {
audioHelper: this._audio,
getUserMedia,
getUserMedia: this._options.getUserMedia || getUserMedia,
isUnifiedPlanDefault: Device._isUnifiedPlanDefault,
onIgnore: (): void => {
this._soundcache.get(Device.SoundName.Incoming).stop();
Expand All @@ -952,6 +952,7 @@ class Device extends EventEmitter {

options = Object.assign({
MediaStream: this._options.MediaStream || rtc.PeerConnection,
RTCPeerConnection: this._options.RTCPeerConnection,
beforeAccept: (currentCall: Call) => {
if (!this._activeCall || this._activeCall === currentCall) {
return;
Expand All @@ -969,7 +970,7 @@ class Device extends EventEmitter {
maxAverageBitrate: this._options.maxAverageBitrate,
preflight: this._options.preflight,
rtcConstraints: this._options.rtcConstraints,
shouldPlayDisconnect: () => this.audio?.disconnect(),
shouldPlayDisconnect: () => this._audio?.disconnect(),
twimlParams,
voiceEventSidGenerator: this._options.voiceEventSidGenerator,
}, options);
Expand All @@ -986,6 +987,12 @@ class Device extends EventEmitter {

const call = new (this._options.Call || Call)(config, options);

this._publisher.info('settings', 'init', {
RTCPeerConnection: !!this._options.RTCPeerConnection,
enumerateDevices: !!this._options.enumerateDevices,
getUserMedia: !!this._options.getUserMedia,
}, call);

call.once('accept', () => {
this._stream.updatePreferredURI(this._preferredURI);
this._removeCall(call);
Expand All @@ -994,7 +1001,7 @@ class Device extends EventEmitter {
this._audio._maybeStartPollingVolume();
}

if (call.direction === Call.CallDirection.Outgoing && this.audio?.outgoing()) {
if (call.direction === Call.CallDirection.Outgoing && this._audio?.outgoing()) {
this._soundcache.get(Device.SoundName.Outgoing).play();
}

Expand Down Expand Up @@ -1204,7 +1211,7 @@ class Device extends EventEmitter {
this._publishNetworkChange();
});

const play = (this.audio?.incoming() && !wasBusy)
const play = (this._audio?.incoming() && !wasBusy)
? () => this._soundcache.get(Device.SoundName.Incoming).play()
: () => Promise.resolve();

Expand Down Expand Up @@ -1310,8 +1317,11 @@ class Device extends EventEmitter {
this._audio = new (this._options.AudioHelper || AudioHelper)(
this._updateSinkIds,
this._updateInputStream,
getUserMedia,
{ audioContext: Device.audioContext },
this._options.getUserMedia || getUserMedia,
{
audioContext: Device.audioContext,
enumerateDevices: this._options.enumerateDevices,
},
);

this._audio.on('deviceChange', (lostActiveDevices: MediaDeviceInfo[]) => {
Expand Down Expand Up @@ -1687,12 +1697,22 @@ namespace Device {
*/
edge?: string[] | string;

/**
* Overrides the native MediaDevices.enumerateDevices API.
*/
enumerateDevices?: any;

/**
* Experimental feature.
* Whether to use ICE Aggressive nomination.
*/
forceAggressiveIceNomination?: boolean;

/**
* Overrides the native MediaDevices.getUserMedia API.
*/
getUserMedia?: any;

/**
* Log level.
*/
Expand Down Expand Up @@ -1724,6 +1744,11 @@ namespace Device {
*/
maxCallSignalingTimeoutMs?: number;

/**
* Overrides the native RTCPeerConnection class.
*/
RTCPeerConnection?: any;

/**
* A mapping of custom sound URLs by sound name.
*/
Expand Down
15 changes: 13 additions & 2 deletions lib/twilio/log.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,13 @@ class Log {
* @param [options] - Optional settings
*/
constructor(options?: LogOptions) {
this._log = (options && options.LogLevelModule ? options.LogLevelModule : LogLevelModule).getLogger(PACKAGE_NAME);
try {
this._log = (options && options.LogLevelModule ? options.LogLevelModule : LogLevelModule).getLogger(PACKAGE_NAME);
} catch {
// tslint:disable-next-line
console.warn('Cannot create custom logger');
this._log = console as any;
}
}

/**
Expand Down Expand Up @@ -93,7 +99,12 @@ class Log {
* Set a default log level to disable all logging below the given level
*/
setDefaultLevel(level: LogLevelModule.LogLevelDesc): void {
this._log.setDefaultLevel(level);
if (this._log.setDefaultLevel) {
this._log.setDefaultLevel(level);
} else {
// tslint:disable-next-line
console.warn('Logger cannot setDefaultLevel');
}
}

/**
Expand Down
64 changes: 48 additions & 16 deletions lib/twilio/rtc/peerconnection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ function PeerConnection(audioHelper, pstream, getUserMedia, options) {
}

function noop() { }
this.onaudio = noop;
this.onopen = noop;
this.onerror = noop;
this.onclose = noop;
Expand Down Expand Up @@ -228,9 +229,14 @@ PeerConnection.prototype._updateInputStreamSource = function(stream) {
this._inputStreamSource.disconnect();
}

this._inputStreamSource = this._audioContext.createMediaStreamSource(stream);
this._inputStreamSource.connect(this._inputAnalyser);
this._inputStreamSource.connect(this._inputAnalyser2);
try {
this._inputStreamSource = this._audioContext.createMediaStreamSource(stream);
this._inputStreamSource.connect(this._inputAnalyser);
this._inputStreamSource.connect(this._inputAnalyser2);
} catch (ex) {
this._log.warn('Unable to update input MediaStreamSource', ex);
this._inputStreamSource = null;
}
};

/**
Expand All @@ -243,9 +249,14 @@ PeerConnection.prototype._updateOutputStreamSource = function(stream) {
this._outputStreamSource.disconnect();
}

this._outputStreamSource = this._audioContext.createMediaStreamSource(stream);
this._outputStreamSource.connect(this._outputAnalyser);
this._outputStreamSource.connect(this._outputAnalyser2);
try {
this._outputStreamSource = this._audioContext.createMediaStreamSource(stream);
this._outputStreamSource.connect(this._outputAnalyser);
this._outputStreamSource.connect(this._outputAnalyser2);
} catch (ex) {
this._log.warn('Unable to update output MediaStreamSource', ex);
this._outputStreamSource = null;
}
};

/**
Expand Down Expand Up @@ -467,15 +478,20 @@ PeerConnection.prototype._updateAudioOutputs = function updateAudioOutputs() {
};

PeerConnection.prototype._createAudio = function createAudio(arr) {
return new Audio(arr);
const audio = new Audio(arr);
this.onaudio(audio);
return audio;
};

PeerConnection.prototype._createAudioOutput = function createAudioOutput(id) {
const dest = this._audioContext.createMediaStreamDestination();
this._mediaStreamSource.connect(dest);
let dest = null;
if (this._mediaStreamSource) {
dest = this._audioContext.createMediaStreamDestination();
this._mediaStreamSource.connect(dest);
}

const audio = this._createAudio();
setAudioSource(audio, dest.stream);
setAudioSource(audio, dest && dest.stream ? dest.stream : this.pcStream);

const self = this;
return audio.setSinkId(id).then(() => audio.play()).then(() => {
Expand Down Expand Up @@ -577,7 +593,12 @@ PeerConnection.prototype._onAddTrack = function onAddTrack(pc, stream) {
audio
});

pc._mediaStreamSource = pc._audioContext.createMediaStreamSource(stream);
try {
pc._mediaStreamSource = pc._audioContext.createMediaStreamSource(stream);
} catch (ex) {
this._log.warn('Unable to create a MediaStreamSource from onAddTrack', ex);
this._mediaStreamSource = null;
}

pc.pcStream = stream;
pc._updateAudioOutputs();
Expand Down Expand Up @@ -630,7 +651,7 @@ PeerConnection.prototype._setEncodingParameters = function(enableDscp) {

PeerConnection.prototype._setupPeerConnection = function(rtcConstraints, rtcConfiguration) {
const self = this;
const version = new (this.options.rtcpcFactory || RTCPC)();
const version = new (this.options.rtcpcFactory || RTCPC)({ RTCPeerConnection: this.options.RTCPeerConnection });
version.create(rtcConstraints, rtcConfiguration);
addStream(version.pc, this.stream);

Expand Down Expand Up @@ -690,10 +711,21 @@ PeerConnection.prototype._setupChannel = function() {
};

// Chrome 72+
pc.onconnectionstatechange = () => {
this._log.info(`pc.connectionState is "${pc.connectionState}"`);
this.onpcconnectionstatechange(pc.connectionState);
this._onMediaConnectionStateChange(pc.connectionState);
pc.onconnectionstatechange = event => {
let state = pc.connectionState;
if (!state && event && event.target) {
// VDI environment
const targetPc = event.target;
state = targetPc.connectionState || targetPc.connectionState_;
this._log.info(`pc.connectionState not detected. Using target PC. State=${state}`);
}
if (!state) {
this._log.warn(`onconnectionstatechange detected but state is "${state}"`);
} else {
this._log.info(`pc.connectionState is "${state}"`);
}
this.onpcconnectionstatechange(state);
this._onMediaConnectionStateChange(state);
};

pc.onicecandidate = event => {
Expand Down
Loading

0 comments on commit 3367771

Please sign in to comment.