Skip to content

Commit

Permalink
fix: experimental option to force single codec preference in the SDP (#…
Browse files Browse the repository at this point in the history 
…1581)

This is a hotfix for platforms that don't respect the ordered codec list(Firefox, Android, Linux, etc.).
We remove all the codecs from the SDP except the one we want to use. This is an opt-in feature hidden behind a toggle in the call's publish options.
To enable it:
```ts
call.updatePublishOptions({
  preferredCodec: 'vp9',
  forceSingleCodec: true,
});
```
  • Loading branch information
@oliverlaz
oliverlaz authored Nov 21, 2024
1 parent e47e046 commit 894a86e
Show file tree
Hide file tree
Showing 6 changed files with 258 additions and 2 deletions.
169 changes: 168 additions & 1 deletion packages/client/src/helpers/__tests__/sdp-munging.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import { describe, expect, it } from 'vitest';
import { enableHighQualityAudio, toggleDtx } from '../sdp-munging';
import {
enableHighQualityAudio,
preserveCodec,
toggleDtx,
} from '../sdp-munging';
import { initialSdp as HQAudioSDP } from './hq-audio-sdp';

describe('sdp-munging', () => {
Expand All @@ -21,4 +25,167 @@ a=maxptime:40`;
expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000');
expect(sdpWithHighQualityAudio).toContain('stereo=1');
});

it('preserves the preferred codec', () => {
const sdp = `v=0
o=- 8608371809202407637 2 IN IP4 127.0.0.1
s=-
t=0 0
a=extmap-allow-mixed
a=msid-semantic: WMS 52fafc21-b8bb-4f4f-8072-86a29cb6590e
a=group:BUNDLE 0
m=video 9 UDP/TLS/RTP/SAVPF 98 100 99 101
c=IN IP4 0.0.0.0
a=rtpmap:98 VP9/90000
a=rtpmap:99 rtx/90000
a=rtpmap:100 VP9/90000
a=rtpmap:101 rtx/90000
a=fmtp:98 profile-id=0
a=fmtp:99 apt=98
a=fmtp:100 profile-id=2
a=fmtp:101 apt=100
a=rtcp:9 IN IP4 0.0.0.0
a=rtcp-fb:98 goog-remb
a=rtcp-fb:98 transport-cc
a=rtcp-fb:98 ccm fir
a=rtcp-fb:98 nack
a=rtcp-fb:98 nack pli
a=rtcp-fb:100 goog-remb
a=rtcp-fb:100 transport-cc
a=rtcp-fb:100 ccm fir
a=rtcp-fb:100 nack
a=rtcp-fb:100 nack pli
a=extmap:1 urn:ietf:params:rtp-hdrext:toffset
a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
a=extmap:3 urn:3gpp:video-orientation
a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type
a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing
a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space
a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid
a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00
a=setup:actpass
a=mid:0
a=msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232
a=sendonly
a=ice-ufrag:LvRk
a=ice-pwd:IpBRr2Rrg9TkOgayjYqALhPY
a=fingerprint:sha-256 18:DE:8F:ED:E6:A2:0C:99:A8:25:AB:C9:F8:3D:91:4C:3E:9F:B4:1F:22:87:A7:3C:85:8F:F3:51:09:A7:E3:FA
a=ice-options:trickle
a=ssrc:3192778601 cname:yYSN5R+RG2j3luO7
a=ssrc:3192778601 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232
a=ssrc:283365205 cname:yYSN5R+RG2j3luO7
a=ssrc:283365205 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232
a=ssrc-group:FID 3192778601 283365205
a=rtcp-mux
a=rtcp-rsize`;
const target = preserveCodec(sdp, '0', {
mimeType: 'video/VP9',
clockRate: 90000,
sdpFmtpLine: 'profile-id=0',
});
expect(target).toContain('VP9');
expect(target).not.toContain('profile-id=2');
});

it('handles ios munging', () => {
const sdp = `v=0
o=- 525780719364332676 2 IN IP4 127.0.0.1
s=-
t=0 0
a=group:BUNDLE 0
a=extmap-allow-mixed
a=msid-semantic: WMS BF3AFE62-88F8-4189-99D7-7CAE159205E3
m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 127 103 35 36 104 105 106
c=IN IP4 0.0.0.0
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:SAkq
a=ice-pwd:FYHHro0VWRO8CjI/M1VG5vRw
a=ice-options:trickle renomination
a=fingerprint:sha-256 03:5B:16:0E:E1:7B:FE:4F:9A:5C:AC:CF:08:21:4B:49:CE:53:79:E6:97:AE:4E:73:F8:43:34:C3:11:F7:6D:E7
a=setup:actpass
a=mid:0
a=extmap:1 urn:ietf:params:rtp-hdrext:toffset
a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
a=extmap:3 urn:3gpp:video-orientation
a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type
a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing
a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space
a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid
a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00
a=sendonly
a=msid:BF3AFE62-88F8-4189-99D7-7CAE159205E3 6013DC02-A0A5-43A9-9D41-9D4A89648A42
a=rtcp-mux
a=rtcp-rsize
a=rtpmap:96 H264/90000
a=rtcp-fb:96 goog-remb
a=rtcp-fb:96 transport-cc
a=rtcp-fb:96 ccm fir
a=rtcp-fb:96 nack
a=rtcp-fb:96 nack pli
a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c29
a=rtpmap:97 rtx/90000
a=fmtp:97 apt=96
a=rtpmap:98 H264/90000
a=rtcp-fb:98 goog-remb
a=rtcp-fb:98 transport-cc
a=rtcp-fb:98 ccm fir
a=rtcp-fb:98 nack
a=rtcp-fb:98 nack pli
a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e029
a=rtpmap:99 rtx/90000
a=fmtp:99 apt=98
a=rtpmap:100 VP8/90000
a=rtcp-fb:100 goog-remb
a=rtcp-fb:100 transport-cc
a=rtcp-fb:100 ccm fir
a=rtcp-fb:100 nack
a=rtcp-fb:100 nack pli
a=rtpmap:101 rtx/90000
a=fmtp:101 apt=100
a=rtpmap:127 VP9/90000
a=rtcp-fb:127 goog-remb
a=rtcp-fb:127 transport-cc
a=rtcp-fb:127 ccm fir
a=rtcp-fb:127 nack
a=rtcp-fb:127 nack pli
a=rtpmap:103 rtx/90000
a=fmtp:103 apt=127
a=rtpmap:35 AV1/90000
a=rtcp-fb:35 goog-remb
a=rtcp-fb:35 transport-cc
a=rtcp-fb:35 ccm fir
a=rtcp-fb:35 nack
a=rtcp-fb:35 nack pli
a=rtpmap:36 rtx/90000
a=fmtp:36 apt=35
a=rtpmap:104 red/90000
a=rtpmap:105 rtx/90000
a=fmtp:105 apt=104
a=rtpmap:106 ulpfec/90000
a=rid:q send
a=rid:h send
a=rid:f send
a=simulcast:send q;h;f`;
const target = preserveCodec(sdp, '0', {
mimeType: 'video/H264',
clockRate: 90000,
sdpFmtpLine:
'profile-level-id=42e029;packetization-mode=1;level-asymmetry-allowed=1',
});
expect(target).toContain('H264');
expect(target).toContain('profile-level-id=42e029');
expect(target).not.toContain('profile-level-id=640c29');
expect(target).not.toContain('VP9');
expect(target).not.toContain('AV1');
});
});
55 changes: 55 additions & 0 deletions packages/client/src/helpers/sdp-munging.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,61 @@ export const toggleDtx = (sdp: string, enable: boolean): string => {
return sdp.replace(opusFmtp.original, newFmtp);
};

/**
* Returns and SDP with all the codecs except the given codec removed.
*/
export const preserveCodec = (
sdp: string,
mid: string,
codec: RTCRtpCodec,
): string => {
const [kind, codecName] = codec.mimeType.toLowerCase().split('/');

const toSet = (fmtpLine: string) =>
new Set(fmtpLine.split(';').map((f) => f.trim().toLowerCase()));

const equal = (a: Set<string>, b: Set<string>) => {
if (a.size !== b.size) return false;
for (const item of a) if (!b.has(item)) return false;
return true;
};

const codecFmtp = toSet(codec.sdpFmtpLine || '');
const parsedSdp = SDP.parse(sdp);
for (const media of parsedSdp.media) {
if (media.type !== kind || String(media.mid) !== mid) continue;

// find the payload id of the desired codec
const payloads = new Set<number>();
for (const rtp of media.rtp) {
if (
rtp.codec.toLowerCase() === codecName &&
media.fmtp.some(
(f) => f.payload === rtp.payload && equal(toSet(f.config), codecFmtp),
)
) {
payloads.add(rtp.payload);
}
}

// find the corresponding rtx codec by matching apt=<preserved-codec-payload>
for (const fmtp of media.fmtp) {
const match = fmtp.config.match(/(apt)=(\d+)/);
if (!match) continue;
const [, , preservedCodecPayload] = match;
if (payloads.has(Number(preservedCodecPayload))) {
payloads.add(fmtp.payload);
}
}

media.rtp = media.rtp.filter((r) => payloads.has(r.payload));
media.fmtp = media.fmtp.filter((f) => payloads.has(f.payload));
media.rtcpFb = media.rtcpFb?.filter((f) => payloads.has(f.payload));
media.payloads = Array.from(payloads).join(' ');
}
return SDP.write(parsedSdp);
};

/**
* Enables high-quality audio through SDP munging for the given trackMid.
*
Expand Down
24 changes: 24 additions & 0 deletions packages/client/src/rtc/Publisher.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { PublishOptions } from '../types';
import {
enableHighQualityAudio,
extractMid,
preserveCodec,
toggleDtx,
} from '../helpers/sdp-munging';
import { Logger } from '../coordinator/connection/types';
Expand Down Expand Up @@ -530,6 +531,12 @@ export class Publisher {
if (this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) {
offer.sdp = this.enableHighQualityAudio(offer.sdp);
}
if (this.isPublishing(TrackType.VIDEO)) {
// Hotfix for platforms that don't respect the ordered codec list
// (Firefox, Android, Linux, etc...).
// We remove all the codecs from the SDP except the one we want to use.
offer.sdp = this.removeUnpreferredCodecs(offer.sdp, TrackType.VIDEO);
}
}

const trackInfos = this.getAnnouncedTracks(offer.sdp);
Expand Down Expand Up @@ -564,6 +571,23 @@ export class Publisher {
);
};

private removeUnpreferredCodecs(sdp: string, trackType: TrackType): string {
const opts = this.publishOptsForTrack.get(trackType);
if (!opts || !opts.forceSingleCodec) return sdp;

const codec = opts.forceCodec || opts.preferredCodec;
const orderedCodecs = this.getCodecPreferences(trackType, codec);
if (!orderedCodecs || orderedCodecs.length === 0) return sdp;

const transceiver = this.transceiverCache.get(trackType);
if (!transceiver) return sdp;

const index = this.transceiverInitOrder.indexOf(trackType);
const mid = extractMid(transceiver, index, sdp);
const [codecToPreserve] = orderedCodecs;
return preserveCodec(sdp, mid, codecToPreserve);
}

private enableHighQualityAudio = (sdp: string) => {
const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO);
if (!transceiver) return sdp;
Expand Down
2 changes: 1 addition & 1 deletion packages/client/src/rtc/codecs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export const getPreferredCodecs = (
}

const sdpFmtpLine = codec.sdpFmtpLine;
if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42e01f')) {
if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42')) {
// this is not the baseline h264 codec, prioritize it lower
partiallyPreferred.push(codec);
continue;
Expand Down
6 changes: 6 additions & 0 deletions packages/client/src/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,12 @@ export type PublishOptions = {
* Use with caution.
*/
forceCodec?: PreferredCodec;
/**
* When using a preferred codec, force the use of a single codec.
* Enabling this, it will remove all other supported codecs from the SDP.
* Defaults to false.
*/
forceSingleCodec?: boolean;
/**
* The preferred scalability to use when publishing the video stream.
* Applicable only for SVC codecs.
Expand Down
4 changes: 4 additions & 0 deletions sample-apps/react/react-dogfood/components/MeetingUI.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => {
| PreferredCodec
| undefined;
const bitrateOverride = router.query['bitrate'] as string | undefined;
const forceSingleCodec = router.query['force_single_codec'] === 'true';
const bitrateFactorOverride = router.query['bitrate_factor'] as
| string
| undefined;
Expand All @@ -72,6 +73,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => {
call.updatePublishOptions({
preferredCodec: 'vp9',
forceCodec: videoCodecOverride,
forceSingleCodec,
scalabilityMode,
preferredBitrate,
bitrateDownscaleFactor: bitrateFactorOverride
Expand All @@ -94,6 +96,8 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => {
bitrateFactorOverride,
bitrateOverride,
call,
forceSingleCodec,
maxSimulcastLayers,
scalabilityMode,
videoCodecOverride,
],
Expand Down

0 comments on commit 894a86e

Please sign in to comment.