From 86cf47ce64c273f67750841f6d0a52f0f697aef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 5 Jan 2023 13:55:50 +0100 Subject: [PATCH 1/3] Remove video track when muting video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 31 ++++++++++++++++++++++++++++--- src/webrtc/groupCall.ts | 5 +++-- src/webrtc/mediaHandler.ts | 34 ++++++++++++++++++++-------------- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d6fde54ed94..4d104ee9bf0 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1280,7 +1280,10 @@ export class MatrixCall extends TypedEventEmitter t.kind === kind)) { + this.peerConn?.removeTrack(sender); + } + } + // Thirdly, we replace the old tracks, if possible. for (const track of stream.getTracks()) { const tKey = getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, track.kind); - const oldSender = this.transceivers.get(tKey)?.sender; + const transceiver = this.transceivers.get(tKey); + const oldSender = transceiver?.sender; let added = false; if (oldSender) { try { @@ -1306,6 +1322,10 @@ export class MatrixCall extends TypedEventEmitter 0) { + canReuseStream = false; + } + if (shouldRequestVideo !== this.localUserMediaStream.getVideoTracks().length > 0) { + canReuseStream = false; + } + // This code checks that the device ID is the same as the localUserMediaStream stream, but we update // the localUserMediaStream whenever the device ID changes (apart from when restoring) so it's not // clear why this would ever be different, unless there's a race. - if (shouldRequestAudio) { - if ( - this.localUserMediaStream.getAudioTracks().length === 0 || - this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput - ) { - canReuseStream = false; - } + if ( + shouldRequestAudio && + this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput + ) { + canReuseStream = false; } - if (shouldRequestVideo) { - if ( - this.localUserMediaStream.getVideoTracks().length === 0 || - this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput - ) { - canReuseStream = false; - } + if ( + shouldRequestVideo && + this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput + ) { + canReuseStream = false; } } else { canReuseStream = false; From 81c2126a69811100d2d4d2cf61df7d1b339e3fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 5 Jan 2023 18:29:41 +0100 Subject: [PATCH 2/3] Fix tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/groupCall.spec.ts | 8 ++++++-- spec/unit/webrtc/mediaHandler.spec.ts | 18 ++++++++---------- src/webrtc/groupCall.ts | 2 ++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 281497bbf62..7d822ec334e 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -810,7 +810,10 @@ describe("Group Call", function () { it("should mute local video when calling setLocalVideoMuted()", async () => { const groupCall = await createAndEnterGroupCall(mockClient, room); - groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); + jest.spyOn(mockClient.getMediaHandler(), "getUserMediaStream"); + jest.spyOn(groupCall, "updateLocalUsermediaStream"); + jest.spyOn(groupCall.localCallFeed!, "setAudioVideoMuted"); + const setAVMutedArray: ((audioMuted: boolean | null, videoMuted: boolean | null) => void)[] = []; const tracksArray: MediaStreamTrack[] = []; const sendMetadataUpdateArray: (() => Promise)[] = []; @@ -824,7 +827,8 @@ describe("Group Call", function () { await groupCall.setLocalVideoMuted(true); groupCall.localCallFeed!.stream.getVideoTracks().forEach((track) => expect(track.enabled).toBe(false)); - expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(null, true); + expect(mockClient.getMediaHandler().getUserMediaStream).toHaveBeenCalledWith(true, false); + expect(groupCall.updateLocalUsermediaStream).toHaveBeenCalled(); setAVMutedArray.forEach((f) => expect(f).toHaveBeenCalledWith(null, true)); tracksArray.forEach((track) => expect(track.enabled).toBe(false)); sendMetadataUpdateArray.forEach((f) => expect(f).toHaveBeenCalled()); diff --git a/spec/unit/webrtc/mediaHandler.spec.ts b/spec/unit/webrtc/mediaHandler.spec.ts index 8a595cdc0a9..1dc84e58550 100644 --- a/spec/unit/webrtc/mediaHandler.spec.ts +++ b/spec/unit/webrtc/mediaHandler.spec.ts @@ -308,20 +308,18 @@ describe("Media Handler", function () { expect(stream2.isCloneOf(stream1)).toEqual(false); }); - it("strips unwanted audio tracks from re-used stream", async () => { - const stream1 = await mediaHandler.getUserMediaStream(true, true); - const stream2 = (await mediaHandler.getUserMediaStream(false, true)) as unknown as MockMediaStream; + it("creates new stream when we no longer want audio", async () => { + await mediaHandler.getUserMediaStream(true, true); + const stream = await mediaHandler.getUserMediaStream(false, true); - expect(stream2.isCloneOf(stream1)).toEqual(true); - expect(stream2.getAudioTracks().length).toEqual(0); + expect(stream.getAudioTracks().length).toEqual(0); }); - it("strips unwanted video tracks from re-used stream", async () => { - const stream1 = await mediaHandler.getUserMediaStream(true, true); - const stream2 = (await mediaHandler.getUserMediaStream(true, false)) as unknown as MockMediaStream; + it("creates new stream when we no longer want video", async () => { + await mediaHandler.getUserMediaStream(true, true); + const stream = await mediaHandler.getUserMediaStream(true, false); - expect(stream2.isCloneOf(stream1)).toEqual(true); - expect(stream2.getVideoTracks().length).toEqual(0); + expect(stream.getVideoTracks().length).toEqual(0); }); }); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3c4ac024eeb..018392cff8e 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -612,6 +612,8 @@ export class GroupCall extends TypedEventEmitter< const stream = await this.client.getMediaHandler().getUserMediaStream(true, !muted); await this.updateLocalUsermediaStream(stream); + this.localCallFeed.setAudioVideoMuted(null, muted); + setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); } else { logger.log(`groupCall ${this.groupCallId} setLocalVideoMuted no stream muted ${muted}`); this.initWithVideoMuted = muted; From bd3c4d648e98aa8961c029d0c68e32f0bfe1fbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 6 Jan 2023 11:29:28 +0100 Subject: [PATCH 3/3] Make stream re-usable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4d104ee9bf0..d37df7cda6a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1370,7 +1370,7 @@ export class MatrixCall extends TypedEventEmitter