Add experimental fallback codec support (#334)

* add experimental polycast encodings * remove unneeded null check * rename to multiCodecSimulcast * capitalize * wip * wip * use backupCodec as indicator for multi codec simulcast * use backup codec options * add default backup encoding * make sure same codec as backup has no effect * use same bitrate as vp8 for h264 * remove vp9 reference and account for #397 * remove comments * remove comments * changeset
livekit · Aug 18, 2022 · 8cb17ec · 8cb17ec
1 parent d6dd20c
commit 8cb17ec
Show file tree

Hide file tree

Showing 6 changed files with 163 additions and 50 deletions.
diff --git a/.changeset/fresh-sheep-burn.md b/.changeset/fresh-sheep-burn.md
@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Add experimental support for fallback codec
diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts
@@ -23,7 +23,9 @@ import LocalTrackPublication from '../track/LocalTrackPublication';
 import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
 import {
   AudioCaptureOptions,
+  BackupVideoCodec,
   CreateLocalTracksOptions,
+  isBackupCodec,
   ScreenShareCaptureOptions,
   ScreenSharePresets,
   TrackPublishOptions,
@@ -35,10 +37,13 @@ import { constraintsForOptions, mergeDefaultOptions } from '../track/utils';
 import { isFireFox, isWeb } from '../utils';
 import Participant from './Participant';
 import { ParticipantTrackPermission, trackPermissionToProto } from './ParticipantTrackPermission';
-import { computeVideoEncodings, mediaTrackToLocalTrack } from './publishUtils';
+import {
+  computeTrackBackupEncodings,
+  computeVideoEncodings,
+  mediaTrackToLocalTrack,
+} from './publishUtils';
 import RemoteParticipant from './RemoteParticipant';
 
-const compatibleCodec = 'vp8';
 export default class LocalParticipant extends Participant {
   audioTracks: Map<string, LocalTrackPublication>;
 
@@ -492,32 +497,25 @@ export default class LocalParticipant extends Participant {
       req.height = height ?? 0;
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (track instanceof LocalVideoTrack) {
-        if (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1') {
+        if (opts?.videoCodec === 'av1') {
           // set scalabilityMode to 'L3T3' by default
           opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+        }
 
-          // add backup codec track
+        // set up backup
+        if (opts.videoCodec && opts.backupCodec && opts.videoCodec !== opts.backupCodec.codec) {
           const simOpts = { ...opts };
           simOpts.simulcast = true;
-          simOpts.scalabilityMode = undefined;
-          simEncodings = computeVideoEncodings(
-            track.source === Track.Source.ScreenShare,
-            width,
-            height,
-            simOpts,
-          );
-        }
+          simEncodings = computeTrackBackupEncodings(track, opts.backupCodec.codec, simOpts);
 
-        // set vp8 codec as backup for any other codecs
-        if (opts.videoCodec && opts.videoCodec !== 'vp8') {
           req.simulcastCodecs = [
             {
               codec: opts.videoCodec,
               cid: track.mediaStreamTrack.id,
               enableSimulcastLayers: true,
             },
             {
-              codec: compatibleCodec,
+              codec: opts.backupCodec.codec,
               cid: '',
               enableSimulcastLayers: true,
             },
@@ -598,16 +596,9 @@ export default class LocalParticipant extends Participant {
    */
   async publishAdditionalCodecForTrack(
     track: LocalTrack | MediaStreamTrack,
-    videoCodec: VideoCodec,
+    videoCodec: BackupVideoCodec,
     options?: TrackPublishOptions,
   ) {
-    const opts: TrackPublishOptions = {
-      ...this.roomOptions?.publishDefaults,
-      ...options,
-    };
-    // clear scalabilityMode setting for backup codec
-    opts.scalabilityMode = undefined;
-    opts.videoCodec = videoCodec;
     // is it not published? if so skip
     let existingPublication: LocalTrackPublication | undefined;
     this.tracks.forEach((publication) => {
@@ -626,17 +617,19 @@ export default class LocalParticipant extends Participant {
       throw new TrackInvalidError('track is not a video track');
     }
 
-    const settings = track.mediaStreamTrack.getSettings();
-    const width = settings.width ?? track.dimensions?.width;
-    const height = settings.height ?? track.dimensions?.height;
+    const opts: TrackPublishOptions = {
+      ...this.roomOptions?.publishDefaults,
+      ...options,
+    };
 
-    const encodings = computeVideoEncodings(
-      track.source === Track.Source.ScreenShare,
-      width,
-      height,
-      opts,
-    );
-    const simulcastTrack = track.addSimulcastTrack(opts.videoCodec, encodings);
+    const encodings = computeTrackBackupEncodings(track, videoCodec, opts);
+    if (!encodings) {
+      log.info(
+        `backup codec has been disabled, ignoring request to add additional codec for track`,
+      );
+      return;
+    }
+    const simulcastTrack = track.addSimulcastTrack(videoCodec, encodings);
     const req = AddTrackRequest.fromPartial({
       cid: simulcastTrack.mediaStreamTrack.id,
       type: Track.kindToProto(track.kind),
@@ -671,18 +664,11 @@ export default class LocalParticipant extends Participant {
       simulcastTrack.mediaStreamTrack,
       transceiverInit,
     );
-    this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
-    track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
+    this.setPreferredCodec(transceiver, track.kind, videoCodec);
+    track.setSimulcastTrackSender(videoCodec, transceiver.sender);
 
-    if (videoCodec === 'av1' && encodings[0]?.maxBitrate) {
-      this.engine.publisher.setTrackCodecBitrate(
-        req.cid,
-        videoCodec,
-        encodings[0].maxBitrate / 1000,
-      );
-    }
     this.engine.negotiate();
-    log.debug(`published ${opts.videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
+    log.debug(`published ${videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
   }
 
   unpublishTrack(
@@ -923,8 +909,10 @@ export default class LocalParticipant extends Participant {
       }
       const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
       for await (const codec of newCodecs) {
-        log.debug(`publish ${codec} for ${pub.videoTrack.sid}`);
-        await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
+        if (isBackupCodec(codec)) {
+          log.debug(`publish ${codec} for ${pub.videoTrack.sid}`);
+          await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
+        }
       }
     } else if (update.subscribedQualities.length > 0) {
       pub.videoTrack?.setPublishingLayers(update.subscribedQualities);

diff --git a/src/room/participant/publishUtils.test.ts b/src/room/participant/publishUtils.test.ts
@@ -114,6 +114,38 @@ describe('computeVideoEncodings', () => {
     expect(encodings![0].maxBitrate).toBe(VideoPresets43.h120.encoding.maxBitrate);
     expect(encodings![0].scaleResolutionDownBy).toBe(1);
   });
+
+  //   it('respects default backup codec encoding', () => {
+  //     const vp8Encodings = computeTrackBackupEncodings(false, 100, 120, { simulcast: true });
+  //     const h264Encodings = computeVideoEncodings(false, 100, 120, {
+  //       simulcast: true,
+  //       videoCodec: 'h264',
+  //     });
+  //     const av1Encodings = computeVideoEncodings(false, 100, 120, {
+  //       simulcast: true,
+  //       videoCodec: 'av1',
+  //     });
+  //     expect(h264Encodings).toHaveLength(1);
+  //     expect(h264Encodings![0].rid).toBe('q');
+  //     expect(h264Encodings![0].maxBitrate).toBe(vp8Encodings[0].maxBitrate! * 1.1);
+  //     expect(av1Encodings![0].maxBitrate).toBe(vp8Encodings[0].maxBitrate! * 0.7);
+  //     expect(h264Encodings![0].scaleResolutionDownBy).toBe(1);
+  //   });
+
+  //   it('respects custom backup codec encoding', () => {
+  //     const encodings = computeVideoEncodings(false, 100, 120, {
+  //       simulcast: true,
+  //       videoCodec: 'h264',
+  //       backupCodec: {
+  //         vp8: { maxBitrate: 1_000 },
+  //         h264: { maxBitrate: 2_000 },
+  //       },
+  //     });
+  //     expect(encodings).toHaveLength(1);
+  //     expect(encodings![0].rid).toBe('q');
+  //     expect(encodings![0].maxBitrate).toBe(2_000);
+  //     expect(encodings![0].scaleResolutionDownBy).toBe(1);
+  //   });
 });
 
 describe('customSimulcastLayers', () => {

diff --git a/src/room/participant/publishUtils.ts b/src/room/participant/publishUtils.ts
@@ -3,13 +3,16 @@ import { TrackInvalidError } from '../errors';
 import LocalAudioTrack from '../track/LocalAudioTrack';
 import LocalVideoTrack from '../track/LocalVideoTrack';
 import {
+  BackupVideoCodec,
   ScreenSharePresets,
   TrackPublishOptions,
+  VideoCodec,
   VideoEncoding,
   VideoPreset,
   VideoPresets,
   VideoPresets43,
 } from '../track/options';
+import { Track } from '../track/Track';
 
 /** @internal */
 export function mediaTrackToLocalTrack(
@@ -61,6 +64,25 @@ export const computeDefaultScreenShareSimulcastPresets = (fromPreset: VideoPrese
   );
 };
 
+// /**
+//  *
+//  * @internal
+//  * @experimental
+//  */
+// const computeDefaultMultiCodecSimulcastEncodings = (width: number, height: number) => {
+//   // use vp8 as a default
+//   const vp8 = determineAppropriateEncoding(false, width, height);
+//   const vp9 = { ...vp8, maxBitrate: vp8.maxBitrate * 0.9 };
+//   const h264 = { ...vp8, maxBitrate: vp8.maxBitrate * 1.1 };
+//   const av1 = { ...vp8, maxBitrate: vp8.maxBitrate * 0.7 };
+//   return {
+//     vp8,
+//     vp9,
+//     h264,
+//     av1,
+//   };
+// };
+
 const videoRids = ['q', 'h', 'f'];
 
 /* @internal */
@@ -71,11 +93,14 @@ export function computeVideoEncodings(
   options?: TrackPublishOptions,
 ): RTCRtpEncodingParameters[] {
   let videoEncoding: VideoEncoding | undefined = options?.videoEncoding;
+
   if (isScreenShare) {
     videoEncoding = options?.screenShareEncoding;
   }
+
   const useSimulcast = options?.simulcast;
   const scalabilityMode = options?.scalabilityMode;
+  const videoCodec = options?.videoCodec;
 
   if ((!videoEncoding && !useSimulcast && !scalabilityMode) || !width || !height) {
     // when we aren't simulcasting or svc, will need to return a single encoding without
@@ -85,7 +110,7 @@ export function computeVideoEncodings(
 
   if (!videoEncoding) {
     // find the right encoding based on width/height
-    videoEncoding = determineAppropriateEncoding(isScreenShare, width, height);
+    videoEncoding = determineAppropriateEncoding(isScreenShare, width, height, videoCodec);
     log.debug('using video encoding', videoEncoding);
   }
 
@@ -96,17 +121,19 @@ export function computeVideoEncodings(
     videoEncoding.maxFramerate,
   );
 
-  log.debug(`scalabilityMode ${scalabilityMode}`);
-  if (scalabilityMode) {
+  if (scalabilityMode && videoCodec === 'av1') {
+    log.debug(`using svc with scalabilityMode ${scalabilityMode}`);
+
     const encodings: RTCRtpEncodingParameters[] = [];
+
     // svc use first encoding as the original, so we sort encoding from high to low
     switch (scalabilityMode) {
       case 'L3T3':
         for (let i = 0; i < 3; i += 1) {
           encodings.push({
             rid: videoRids[2 - i],
             scaleResolutionDownBy: 2 ** i,
-            maxBitrate: videoEncoding ? videoEncoding.maxBitrate / 3 ** i : 0,
+            maxBitrate: videoEncoding.maxBitrate / 3 ** i,
             /* @ts-ignore */
             maxFramerate: original.encoding.maxFramerate,
             /* @ts-ignore */
@@ -162,11 +189,45 @@ export function computeVideoEncodings(
   return encodingsFromPresets(width, height, [original]);
 }
 
+export function computeTrackBackupEncodings(
+  track: LocalVideoTrack,
+  videoCodec: BackupVideoCodec,
+  opts: TrackPublishOptions,
+) {
+  if (!opts.backupCodec || opts.backupCodec.codec === opts.videoCodec) {
+    // backup codec publishing is disabled
+    return;
+  }
+  if (videoCodec !== opts.backupCodec.codec) {
+    log.warn('requested a different codec than specified as backup', {
+      serverRequested: videoCodec,
+      backup: opts.backupCodec.codec,
+    });
+  }
+
+  opts.videoCodec = videoCodec;
+  // use backup encoding setting as videoEncoding for backup codec publishing
+  opts.videoEncoding = opts.backupCodec.encoding;
+
+  const settings = track.mediaStreamTrack.getSettings();
+  const width = settings.width ?? track.dimensions?.width;
+  const height = settings.height ?? track.dimensions?.height;
+
+  const encodings = computeVideoEncodings(
+    track.source === Track.Source.ScreenShare,
+    width,
+    height,
+    opts,
+  );
+  return encodings;
+}
+
 /* @internal */
 export function determineAppropriateEncoding(
   isScreenShare: boolean,
   width: number,
   height: number,
+  codec?: VideoCodec,
 ): VideoEncoding {
   const presets = presetsForResolution(isScreenShare, width, height);
   let { encoding } = presets[0];
@@ -181,6 +242,18 @@ export function determineAppropriateEncoding(
       break;
     }
   }
+  // presets are based on the assumption of vp8 as a codec
+  // for other codecs we adjust the maxBitrate if no specific videoEncoding has been provided
+  // TODO make the bitrate multipliers configurable per codec
+  if (codec) {
+    switch (codec) {
+      case 'av1':
+        encoding.maxBitrate = encoding.maxBitrate * 0.7;
+        break;
+      default:
+        break;
+    }
+  }
 
   return encoding;
 }

diff --git a/src/room/track/defaults.ts b/src/room/track/defaults.ts
@@ -14,6 +14,7 @@ export const publishDefaults: TrackPublishDefaults = {
   screenShareEncoding: ScreenSharePresets.h1080fps15.encoding,
   stopMicTrackOnMute: false,
   videoCodec: 'vp8',
+  backupCodec: { codec: 'vp8', encoding: VideoPresets.h540.encoding },
 };
 
 export const audioDefaults: AudioCaptureOptions = {

diff --git a/src/room/track/options.ts b/src/room/track/options.ts
@@ -6,6 +6,11 @@ export interface TrackPublishDefaults {
    */
   videoEncoding?: VideoEncoding;
 
+  /**
+   * @experimental
+   */
+  backupCodec?: { codec: BackupVideoCodec; encoding: VideoEncoding } | false;
+
   /**
    * encoding parameters for screen share track
    */
@@ -205,7 +210,16 @@ export interface AudioPreset {
   maxBitrate: number;
 }
 
-export type VideoCodec = 'vp8' | 'h264' | 'av1' | 'vp9';
+const codecs = ['vp8', 'h264', 'av1'] as const;
+const backupCodecs = ['vp8', 'h264'] as const;
+
+export type VideoCodec = typeof codecs[number];
+
+export type BackupVideoCodec = typeof backupCodecs[number];
+
+export function isBackupCodec(codec: string): codec is BackupVideoCodec {
+  return !!backupCodecs.find((backup) => backup === codec);
+}
 
 /**
  * scalability modes for svc, only supprot l3t3 now.