diff --git a/.changeset/purple-coats-attend.md b/.changeset/purple-coats-attend.md new file mode 100644 index 0000000000..df461a3a34 --- /dev/null +++ b/.changeset/purple-coats-attend.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +Improve handling of incompatible published codecs diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index cc0fdac2c0..2f278ddf49 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -11,6 +11,7 @@ import LocalTrack from '../room/track/LocalTrack'; import type RemoteTrack from '../room/track/RemoteTrack'; import type { Track } from '../room/track/Track'; import type { VideoCodec } from '../room/track/options'; +import { mimeTypeToVideoCodecString } from '../room/track/utils'; import type { BaseKeyProvider } from './KeyProvider'; import { E2EE_FLAG } from './constants'; import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events'; @@ -28,7 +29,7 @@ import type { SifTrailerMessage, UpdateCodecMessage, } from './types'; -import { isE2EESupported, isScriptTransformSupported, mimeTypeToVideoCodecString } from './utils'; +import { isE2EESupported, isScriptTransformSupported } from './utils'; /** * @experimental diff --git a/src/e2ee/utils.ts b/src/e2ee/utils.ts index e7f7defce4..62b80555e6 100644 --- a/src/e2ee/utils.ts +++ b/src/e2ee/utils.ts @@ -1,5 +1,3 @@ -import { videoCodecs } from '../room/track/options'; -import type { VideoCodec } from '../room/track/options'; import { ENCRYPTION_ALGORITHM } from './constants'; export function isE2EESupported() { @@ -116,14 +114,6 @@ export function createE2EEKey(): Uint8Array { return window.crypto.getRandomValues(new Uint8Array(32)); } -export function mimeTypeToVideoCodecString(mimeType: string) { - const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec; - if (!videoCodecs.includes(codec)) { - throw Error(`Video codec not supported: ${codec}`); - } - return codec; -} - /** * Ratchets a key. See * https://tools.ietf.org/html/draft-omara-sframe-00#section-4.3.5.1 diff --git a/src/room/defaults.ts b/src/room/defaults.ts index f6312981d0..5577d6cbc1 100644 --- a/src/room/defaults.ts +++ b/src/room/defaults.ts @@ -1,11 +1,13 @@ import type { InternalRoomConnectOptions, InternalRoomOptions } from '../options'; import DefaultReconnectPolicy from './DefaultReconnectPolicy'; -import { AudioPresets, ScreenSharePresets, VideoPresets } from './track/options'; import type { AudioCaptureOptions, TrackPublishDefaults, VideoCaptureOptions, } from './track/options'; +import { AudioPresets, ScreenSharePresets, VideoPresets } from './track/options'; + +export const defaultVideoCodec = 'vp8'; export const publishDefaults: TrackPublishDefaults = { /** @@ -19,7 +21,7 @@ export const publishDefaults: TrackPublishDefaults = { simulcast: true, screenShareEncoding: ScreenSharePresets.h1080fps15.encoding, stopMicTrackOnMute: false, - videoCodec: 'vp8', + videoCodec: defaultVideoCodec, backupCodec: false, } as const; diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index e93d8024a9..4a26eb2a8d 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -18,6 +18,7 @@ import { TrackUnpublishedResponse, } from '../../proto/livekit_rtc_pb'; import type RTCEngine from '../RTCEngine'; +import { defaultVideoCodec } from '../defaults'; import { DeviceUnsupportedError, TrackInvalidError, UnexpectedConnectionState } from '../errors'; import { EngineEvent, ParticipantEvent, TrackEvent } from '../events'; import LocalAudioTrack from '../track/LocalAudioTrack'; @@ -33,10 +34,11 @@ import type { TrackPublishOptions, VideoCaptureOptions, } from '../track/options'; -import { VideoPresets, isBackupCodec, isCodecEqual } from '../track/options'; +import { VideoPresets, isBackupCodec } from '../track/options'; import { constraintsForOptions, mergeDefaultOptions, + mimeTypeToVideoCodecString, screenCaptureToDisplayMediaStreamOptions, } from '../track/utils'; import type { DataPublishOptions } from '../types'; @@ -629,6 +631,10 @@ export default class LocalParticipant extends Participant { if (opts.videoCodec === 'vp9' && !supportsVP9()) { opts.videoCodec = undefined; } + if (opts.videoCodec === undefined) { + opts.videoCodec = defaultVideoCodec; + } + const videoCodec = opts.videoCodec; // handle track actions track.on(TrackEvent.Muted, this.onTrackMuted); @@ -654,7 +660,6 @@ export default class LocalParticipant extends Participant { // compute encodings and layers for video let encodings: RTCRtpEncodingParameters[] | undefined; - let simEncodings: RTCRtpEncodingParameters[] | undefined; if (track.kind === Track.Kind.Video) { let dims: Track.Dimensions = { width: 0, @@ -679,50 +684,40 @@ export default class LocalParticipant extends Participant { req.height = dims.height; // for svc codecs, disable simulcast and use vp8 for backup codec if (track instanceof LocalVideoTrack) { - if (isSVCCodec(opts.videoCodec)) { + if (isSVCCodec(videoCodec)) { // vp9 svc with screenshare has problem to encode, always use L1T3 here - if (track.source === Track.Source.ScreenShare && opts.videoCodec === 'vp9') { + if (track.source === Track.Source.ScreenShare && videoCodec === 'vp9') { opts.scalabilityMode = 'L1T3'; } // set scalabilityMode to 'L3T3_KEY' by default opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY'; } + req.simulcastCodecs = [ + new SimulcastCodec({ + codec: videoCodec, + cid: track.mediaStreamTrack.id, + }), + ]; + // set up backup - if (opts.videoCodec && opts.backupCodec && opts.videoCodec !== opts.backupCodec.codec) { + if (opts.backupCodec && videoCodec !== opts.backupCodec.codec) { if (!this.roomOptions.dynacast) { this.roomOptions.dynacast = true; } - const simOpts = { ...opts }; - simOpts.simulcast = true; - simEncodings = computeTrackBackupEncodings(track, opts.backupCodec.codec, simOpts); - - req.simulcastCodecs = [ - new SimulcastCodec({ - codec: opts.videoCodec, - cid: track.mediaStreamTrack.id, - }), + req.simulcastCodecs.push( new SimulcastCodec({ codec: opts.backupCodec.codec, cid: '', }), - ]; - } else if (opts.videoCodec) { - // pass codec info to sfu so it can prefer codec for the client which don't support - // setCodecPreferences - req.simulcastCodecs = [ - new SimulcastCodec({ - codec: opts.videoCodec, - cid: track.mediaStreamTrack.id, - }), - ]; + ); } } encodings = computeVideoEncodings( track.source === Track.Source.ScreenShare, - dims.width, - dims.height, + req.width, + req.height, opts, ); req.layers = videoLayersFromEncodings( @@ -746,30 +741,28 @@ export default class LocalParticipant extends Participant { } const ti = await this.engine.addTrack(req); - let primaryCodecSupported = false; - let backupCodecSupported = false; - ti.codecs.forEach((c) => { - if (isCodecEqual(c.mimeType, opts.videoCodec)) { - primaryCodecSupported = true; - } else if (opts.backupCodec && isCodecEqual(c.mimeType, opts.backupCodec.codec)) { - backupCodecSupported = true; + // server might not support the codec the client has requested, in that case, fallback + // to a supported codec + let primaryCodecMime: string | undefined; + ti.codecs.forEach((codec) => { + if (primaryCodecMime === undefined) { + primaryCodecMime = codec.mimeType; } }); - - if (req.simulcastCodecs.length > 0) { - if (!primaryCodecSupported && !backupCodecSupported) { - throw Error('cannot publish track, codec not supported by server'); - } - - if (!primaryCodecSupported && opts.backupCodec) { - const backupCodec = opts.backupCodec; - opts = { ...opts }; - log.debug( - `primary codec ${opts.videoCodec} not supported, fallback to ${backupCodec.codec}`, + if (primaryCodecMime && track.kind === Track.Kind.Video) { + const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime); + if (updatedCodec !== videoCodec) { + log.debug('falling back to server selected codec', { codec: updatedCodec }); + /* @ts-ignore */ + opts.videoCodec = updatedCodec; + + // recompute encodings since bitrates/etc could have changed + encodings = computeVideoEncodings( + track.source === Track.Source.ScreenShare, + req.width, + req.height, + opts, ); - opts.videoCodec = backupCodec.codec; - opts.videoEncoding = backupCodec.encoding; - encodings = simEncodings; } } @@ -783,13 +776,12 @@ export default class LocalParticipant extends Participant { } log.debug(`publishing ${track.kind} with encodings`, { encodings, trackInfo: ti }); - // store RTPSender track.sender = await this.engine.createSender(track, opts, encodings); if (encodings) { if (isFireFox() && track.kind === Track.Kind.Audio) { /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1, - livekit-server uses maxaveragebitrate=510000in the answer sdp to permit client to + livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to publish high quality audio track. But firefox always uses this value as the actual bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly. So the client need to modify maxaverragebitrates in answer sdp to user provided value to diff --git a/src/room/track/LocalVideoTrack.ts b/src/room/track/LocalVideoTrack.ts index 72498ca1db..ed27976be4 100644 --- a/src/room/track/LocalVideoTrack.ts +++ b/src/room/track/LocalVideoTrack.ts @@ -284,7 +284,8 @@ export default class LocalVideoTrack extends LocalTrack { /** * @internal - * Sets codecs that should be publishing + * Sets codecs that should be publishing, returns new codecs that have not yet + * been published */ async setPublishingCodecs(codecs: SubscribedCodec[]): Promise { log.debug('setting publishing codecs', { diff --git a/src/room/track/options.ts b/src/room/track/options.ts index 98ca822b25..47eb9ffe1c 100644 --- a/src/room/track/options.ts +++ b/src/room/track/options.ts @@ -301,13 +301,6 @@ export function isBackupCodec(codec: string): codec is BackupVideoCodec { return !!backupCodecs.find((backup) => backup === codec); } -export function isCodecEqual(c1: string | undefined, c2: string | undefined): boolean { - return ( - c1?.toLowerCase().replace(/audio\/|video\//y, '') === - c2?.toLowerCase().replace(/audio\/|video\//y, '') - ); -} - /** * scalability modes for svc. */ diff --git a/src/room/track/utils.ts b/src/room/track/utils.ts index d6faacabcd..30345ece0a 100644 --- a/src/room/track/utils.ts +++ b/src/room/track/utils.ts @@ -1,10 +1,12 @@ import { isSafari, sleep } from '../utils'; import { Track } from './Track'; -import type { - AudioCaptureOptions, - CreateLocalTracksOptions, - ScreenShareCaptureOptions, - VideoCaptureOptions, +import { + type AudioCaptureOptions, + type CreateLocalTracksOptions, + type ScreenShareCaptureOptions, + type VideoCaptureOptions, + VideoCodec, + videoCodecs, } from './options'; import type { AudioTrack } from './types'; @@ -181,3 +183,11 @@ export function screenCaptureToDisplayMediaStreamOptions( systemAudio: options.systemAudio, }; } + +export function mimeTypeToVideoCodecString(mimeType: string) { + const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec; + if (!videoCodecs.includes(codec)) { + throw Error(`Video codec not supported: ${codec}`); + } + return codec; +}