From 894a86e407dc0dd36b7463bb964c86da0c3055d1 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 21 Nov 2024 15:43:17 +0100 Subject: [PATCH] fix: experimental option to force single codec preference in the SDP (#1581) This is a hotfix for platforms that don't respect the ordered codec list(Firefox, Android, Linux, etc.). We remove all the codecs from the SDP except the one we want to use. This is an opt-in feature hidden behind a toggle in the call's publish options. To enable it: ```ts call.updatePublishOptions({ preferredCodec: 'vp9', forceSingleCodec: true, }); ``` --- .../src/helpers/__tests__/sdp-munging.test.ts | 169 +++++++++++++++++- packages/client/src/helpers/sdp-munging.ts | 55 ++++++ packages/client/src/rtc/Publisher.ts | 24 +++ packages/client/src/rtc/codecs.ts | 2 +- packages/client/src/types.ts | 6 + .../react-dogfood/components/MeetingUI.tsx | 4 + 6 files changed, 258 insertions(+), 2 deletions(-) diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts index 37dae3d1fb..922f97f1ba 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/helpers/__tests__/sdp-munging.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { enableHighQualityAudio, toggleDtx } from '../sdp-munging'; +import { + enableHighQualityAudio, + preserveCodec, + toggleDtx, +} from '../sdp-munging'; import { initialSdp as HQAudioSDP } from './hq-audio-sdp'; describe('sdp-munging', () => { @@ -21,4 +25,167 @@ a=maxptime:40`; expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); expect(sdpWithHighQualityAudio).toContain('stereo=1'); }); + + it('preserves the preferred codec', () => { + const sdp = `v=0 +o=- 8608371809202407637 2 IN IP4 127.0.0.1 +s=- +t=0 0 +a=extmap-allow-mixed +a=msid-semantic: WMS 52fafc21-b8bb-4f4f-8072-86a29cb6590e +a=group:BUNDLE 0 +m=video 9 UDP/TLS/RTP/SAVPF 98 100 99 101 +c=IN IP4 0.0.0.0 +a=rtpmap:98 VP9/90000 +a=rtpmap:99 rtx/90000 +a=rtpmap:100 VP9/90000 +a=rtpmap:101 rtx/90000 +a=fmtp:98 profile-id=0 +a=fmtp:99 apt=98 +a=fmtp:100 profile-id=2 +a=fmtp:101 apt=100 +a=rtcp:9 IN IP4 0.0.0.0 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension +a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00 +a=setup:actpass +a=mid:0 +a=msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232 +a=sendonly +a=ice-ufrag:LvRk +a=ice-pwd:IpBRr2Rrg9TkOgayjYqALhPY +a=fingerprint:sha-256 18:DE:8F:ED:E6:A2:0C:99:A8:25:AB:C9:F8:3D:91:4C:3E:9F:B4:1F:22:87:A7:3C:85:8F:F3:51:09:A7:E3:FA +a=ice-options:trickle +a=ssrc:3192778601 cname:yYSN5R+RG2j3luO7 +a=ssrc:3192778601 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232 +a=ssrc:283365205 cname:yYSN5R+RG2j3luO7 +a=ssrc:283365205 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232 +a=ssrc-group:FID 3192778601 283365205 +a=rtcp-mux +a=rtcp-rsize`; + const target = preserveCodec(sdp, '0', { + mimeType: 'video/VP9', + clockRate: 90000, + sdpFmtpLine: 'profile-id=0', + }); + expect(target).toContain('VP9'); + expect(target).not.toContain('profile-id=2'); + }); + + it('handles ios munging', () => { + const sdp = `v=0 +o=- 525780719364332676 2 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 +a=extmap-allow-mixed +a=msid-semantic: WMS BF3AFE62-88F8-4189-99D7-7CAE159205E3 +m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 127 103 35 36 104 105 106 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:SAkq +a=ice-pwd:FYHHro0VWRO8CjI/M1VG5vRw +a=ice-options:trickle renomination +a=fingerprint:sha-256 03:5B:16:0E:E1:7B:FE:4F:9A:5C:AC:CF:08:21:4B:49:CE:53:79:E6:97:AE:4E:73:F8:43:34:C3:11:F7:6D:E7 +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension +a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00 +a=sendonly +a=msid:BF3AFE62-88F8-4189-99D7-7CAE159205E3 6013DC02-A0A5-43A9-9D41-9D4A89648A42 +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 H264/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c29 +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:98 H264/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e029 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP8/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:127 VP9/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=127 +a=rtpmap:35 AV1/90000 +a=rtcp-fb:35 goog-remb +a=rtcp-fb:35 transport-cc +a=rtcp-fb:35 ccm fir +a=rtcp-fb:35 nack +a=rtcp-fb:35 nack pli +a=rtpmap:36 rtx/90000 +a=fmtp:36 apt=35 +a=rtpmap:104 red/90000 +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 ulpfec/90000 +a=rid:q send +a=rid:h send +a=rid:f send +a=simulcast:send q;h;f`; + const target = preserveCodec(sdp, '0', { + mimeType: 'video/H264', + clockRate: 90000, + sdpFmtpLine: + 'profile-level-id=42e029;packetization-mode=1;level-asymmetry-allowed=1', + }); + expect(target).toContain('H264'); + expect(target).toContain('profile-level-id=42e029'); + expect(target).not.toContain('profile-level-id=640c29'); + expect(target).not.toContain('VP9'); + expect(target).not.toContain('AV1'); + }); }); diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index b8bdfd66af..317c1a8e7a 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -129,6 +129,61 @@ export const toggleDtx = (sdp: string, enable: boolean): string => { return sdp.replace(opusFmtp.original, newFmtp); }; +/** + * Returns and SDP with all the codecs except the given codec removed. + */ +export const preserveCodec = ( + sdp: string, + mid: string, + codec: RTCRtpCodec, +): string => { + const [kind, codecName] = codec.mimeType.toLowerCase().split('/'); + + const toSet = (fmtpLine: string) => + new Set(fmtpLine.split(';').map((f) => f.trim().toLowerCase())); + + const equal = (a: Set, b: Set) => { + if (a.size !== b.size) return false; + for (const item of a) if (!b.has(item)) return false; + return true; + }; + + const codecFmtp = toSet(codec.sdpFmtpLine || ''); + const parsedSdp = SDP.parse(sdp); + for (const media of parsedSdp.media) { + if (media.type !== kind || String(media.mid) !== mid) continue; + + // find the payload id of the desired codec + const payloads = new Set(); + for (const rtp of media.rtp) { + if ( + rtp.codec.toLowerCase() === codecName && + media.fmtp.some( + (f) => f.payload === rtp.payload && equal(toSet(f.config), codecFmtp), + ) + ) { + payloads.add(rtp.payload); + } + } + + // find the corresponding rtx codec by matching apt= + for (const fmtp of media.fmtp) { + const match = fmtp.config.match(/(apt)=(\d+)/); + if (!match) continue; + const [, , preservedCodecPayload] = match; + if (payloads.has(Number(preservedCodecPayload))) { + payloads.add(fmtp.payload); + } + } + + media.rtp = media.rtp.filter((r) => payloads.has(r.payload)); + media.fmtp = media.fmtp.filter((f) => payloads.has(f.payload)); + media.rtcpFb = media.rtcpFb?.filter((f) => payloads.has(f.payload)); + media.payloads = Array.from(payloads).join(' '); + } + return SDP.write(parsedSdp); +}; + /** * Enables high-quality audio through SDP munging for the given trackMid. * diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index b223dfc4e0..f479101e53 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -20,6 +20,7 @@ import { PublishOptions } from '../types'; import { enableHighQualityAudio, extractMid, + preserveCodec, toggleDtx, } from '../helpers/sdp-munging'; import { Logger } from '../coordinator/connection/types'; @@ -530,6 +531,12 @@ export class Publisher { if (this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { offer.sdp = this.enableHighQualityAudio(offer.sdp); } + if (this.isPublishing(TrackType.VIDEO)) { + // Hotfix for platforms that don't respect the ordered codec list + // (Firefox, Android, Linux, etc...). + // We remove all the codecs from the SDP except the one we want to use. + offer.sdp = this.removeUnpreferredCodecs(offer.sdp, TrackType.VIDEO); + } } const trackInfos = this.getAnnouncedTracks(offer.sdp); @@ -564,6 +571,23 @@ export class Publisher { ); }; + private removeUnpreferredCodecs(sdp: string, trackType: TrackType): string { + const opts = this.publishOptsForTrack.get(trackType); + if (!opts || !opts.forceSingleCodec) return sdp; + + const codec = opts.forceCodec || opts.preferredCodec; + const orderedCodecs = this.getCodecPreferences(trackType, codec); + if (!orderedCodecs || orderedCodecs.length === 0) return sdp; + + const transceiver = this.transceiverCache.get(trackType); + if (!transceiver) return sdp; + + const index = this.transceiverInitOrder.indexOf(trackType); + const mid = extractMid(transceiver, index, sdp); + const [codecToPreserve] = orderedCodecs; + return preserveCodec(sdp, mid, codecToPreserve); + } + private enableHighQualityAudio = (sdp: string) => { const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO); if (!transceiver) return sdp; diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 8c498c029c..01a8707f9d 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -50,7 +50,7 @@ export const getPreferredCodecs = ( } const sdpFmtpLine = codec.sdpFmtpLine; - if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42e01f')) { + if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42')) { // this is not the baseline h264 codec, prioritize it lower partiallyPreferred.push(codec); continue; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 3cb8a59858..9d951b3ba2 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -167,6 +167,12 @@ export type PublishOptions = { * Use with caution. */ forceCodec?: PreferredCodec; + /** + * When using a preferred codec, force the use of a single codec. + * Enabling this, it will remove all other supported codecs from the SDP. + * Defaults to false. + */ + forceSingleCodec?: boolean; /** * The preferred scalability to use when publishing the video stream. * Applicable only for SVC codecs. diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 7b74d416d6..34176ee561 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -50,6 +50,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { | PreferredCodec | undefined; const bitrateOverride = router.query['bitrate'] as string | undefined; + const forceSingleCodec = router.query['force_single_codec'] === 'true'; const bitrateFactorOverride = router.query['bitrate_factor'] as | string | undefined; @@ -72,6 +73,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { call.updatePublishOptions({ preferredCodec: 'vp9', forceCodec: videoCodecOverride, + forceSingleCodec, scalabilityMode, preferredBitrate, bitrateDownscaleFactor: bitrateFactorOverride @@ -94,6 +96,8 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { bitrateFactorOverride, bitrateOverride, call, + forceSingleCodec, + maxSimulcastLayers, scalabilityMode, videoCodecOverride, ],