Parallelize key and segment requests

api-extractor/report/hls.js.api.md

@@ -724,6 +724,8 @@ export interface FragLoadingData {
 export class Fragment extends BaseSegment {
  constructor(type: PlaylistLevelType, baseurl: string);
  // (undocumented)
+ abortRequests(): void;
+ // (undocumented)
  appendedPTS?: number;
  // (undocumented)
  bitrateTest: boolean;
@@ -753,6 +755,8 @@ export class Fragment extends BaseSegment {
  // (undocumented)
  initSegment: Fragment | null;
  // (undocumented)
+ keyLoader: Loader<KeyLoaderContext> | null;
+ // (undocumented)
  level: number;
  // (undocumented)
  levelkey?: LevelKey;
@@ -1213,6 +1217,12 @@ export interface KeyLoadedData {
  frag: Fragment;
 }
 
+// Warning: (ae-missing-release-tag) "KeyLoaderContext" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface KeyLoaderContext extends FragmentLoaderContext {
+}
+
 // Warning: (ae-missing-release-tag) "KeyLoadingData" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

src/controller/abr-controller.ts

@@ -198,9 +198,9 @@ class AbrController implements ComponentAPI {
  hls.nextLoadLevel = nextLoadLevel;
  this.bwEstimator.sample(requestDelay, stats.loaded);
  this.clearTimer();
- if (frag.loader) {
+ if (frag.loader || frag.keyLoader) {
  this.fragCurrent = this.partCurrent = null;
- frag.loader.abort();
+ frag.abortRequests();
  }
  hls.trigger(Events.FRAG_LOAD_EMERGENCY_ABORTED, { frag, part, stats });
  }

src/controller/audio-stream-controller.ts

@@ -363,11 +363,7 @@ class AudioStreamController
  return;
  }
 
- if (frag.decryptdata?.keyFormat === 'identity' && !frag.decryptdata?.key) {
- this.loadKey(frag, trackDetails);
- } else {
- this.loadFragment(frag, trackDetails, targetBufferTime);
- }
+ this.loadFragment(frag, trackDetails, targetBufferTime);
  }
 
  protected getMaxBufferLength(mainBufferLength?: number): number {
@@ -400,8 +396,8 @@ class AudioStreamController
  this.trackId = data.id;
  const { fragCurrent } = this;
 
- if (fragCurrent?.loader) {
- fragCurrent.loader.abort();
+ if (fragCurrent) {
+ fragCurrent.abortRequests();
  }
  this.fragCurrent = null;
  this.clearWaitingFragment();
@@ -827,7 +823,7 @@ class AudioStreamController
  fragState === FragmentState.PARTIAL
  ) {
  if (frag.sn === 'initSegment') {
- this._loadInitSegment(frag);
+ this._loadInitSegment(frag, trackDetails);
  } else if (trackDetails.live && !Number.isFinite(this.initPTS[frag.cc])) {
  this.log(
  `Waiting for video PTS in continuity counter ${frag.cc} of live stream before loading audio fragment ${frag.sn} of level ${this.trackId}`

src/controller/base-stream-controller.ts

@@ -23,6 +23,7 @@ import FragmentLoader, {
  FragmentLoadProgressCallback,
  LoadError,
 } from '../loader/fragment-loader';
+import KeyLoader from '../loader/key-loader';
 import { LevelDetails } from '../loader/level-details';
 import Decrypter from '../crypt/decrypter';
 import TimeRanges from '../utils/time-ranges';
@@ -86,7 +87,8 @@ export default class BaseStreamController
  protected fragLoadError: number = 0;
  protected retryDate: number = 0;
  protected levels: Array<Level> | null = null;
- protected fragmentLoader!: FragmentLoader;
+ protected fragmentLoader: FragmentLoader;
+ protected keyLoader: KeyLoader;
  protected levelLastLoaded: number | null = null;
  protected startFragRequested: boolean = false;
  protected decrypter: Decrypter;
@@ -105,10 +107,10 @@ export default class BaseStreamController
  this.warn = logger.warn.bind(logger, `${logPrefix}:`);
  this.hls = hls;
  this.fragmentLoader = new FragmentLoader(hls.config);
+ this.keyLoader = new KeyLoader(hls.config);
  this.fragmentTracker = fragmentTracker;
  this.config = hls.config;
  this.decrypter = new Decrypter(hls as HlsEventEmitter, hls.config);
- hls.on(Events.KEY_LOADED, this.onKeyLoaded, this);
  hls.on(Events.LEVEL_SWITCHING, this.onLevelSwitching, this);
  }
 
@@ -123,8 +125,10 @@ export default class BaseStreamController
 
  public stopLoad() {
  this.fragmentLoader.abort();
+ this.keyLoader.abort();
  const frag = this.fragCurrent;
  if (frag) {
+ frag.abortRequests();
  this.fragmentTracker.removeFragment(frag);
  }
  this.resetTransmuxer();
@@ -238,7 +242,7 @@ export default class BaseStreamController
  this.log(
  'seeking outside of buffer while fragment load in progress, cancel fragment load'
  );
- fragCurrent.loader.abort();
+ fragCurrent.abortRequests();
  }
  this.resetLoadingState();
  }
@@ -262,21 +266,6 @@ export default class BaseStreamController
  this.startPosition = this.lastCurrentTime = 0;
  }
 
- onKeyLoaded(event: Events.KEY_LOADED, data: KeyLoadedData) {
- if (
- this.state !== State.KEY_LOADING ||
- data.frag !== this.fragCurrent ||
- !this.levels
- ) {
- return;
- }
- this.state = State.IDLE;
- const levelDetails = this.levels[data.frag.level].details;
- if (levelDetails) {
- this.loadFragment(data.frag, levelDetails, data.frag.start);
- }
- }
-
  protected onLevelSwitching(
  event: Events.LEVEL_SWITCHING,
  data: LevelSwitchingData
@@ -291,11 +280,13 @@ export default class BaseStreamController
 
  protected onHandlerDestroyed() {
  this.state = State.STOPPED;
- this.hls.off(Events.KEY_LOADED, this.onKeyLoaded, this);
  this.hls.off(Events.LEVEL_SWITCHING, this.onLevelSwitching, this);
  if (this.fragmentLoader) {
  this.fragmentLoader.destroy();
  }
+ if (this.keyLoader) {
+ this.keyLoader.destroy();
+ }
  if (this.decrypter) {
  this.decrypter.destroy();
  }
@@ -304,23 +295,13 @@ export default class BaseStreamController
  this.log =
  this.warn =
  this.decrypter =
+ this.keyLoader =
  this.fragmentLoader =
  this.fragmentTracker =
  null as any;
  super.onHandlerDestroyed();
  }
 
- protected loadKey(frag: Fragment, details: LevelDetails) {
- this.log(
- `Loading key for ${frag.sn} of [${details.startSN}-${details.endSN}], ${
- this.logPrefix === '[stream-controller]' ? 'level' : 'track'
- } ${frag.level}`
- );
- this.state = State.KEY_LOADING;
- this.fragCurrent = frag;
- this.hls.trigger(Events.KEY_LOADING, { frag });
- }
-
  protected loadFragment(
  frag: Fragment,
  levelDetails: LevelDetails,
@@ -402,8 +383,8 @@ export default class BaseStreamController
  this.hls.trigger(Events.BUFFER_FLUSHING, flushScope);
  }
 
- protected _loadInitSegment(frag: Fragment) {
- this._doFragLoad(frag)
+ protected _loadInitSegment(frag: Fragment, details: LevelDetails) {
+ this._doFragLoad(frag, details)
  .then((data) => {
  if (!data || this.fragContextChanged(frag) || !this.levels) {
  throw new Error('init load aborted');
@@ -554,17 +535,39 @@ export default class BaseStreamController
  }
 
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
- protected _handleFragmentLoadProgress(frag: FragLoadedData) {}
+ protected _handleFragmentLoadProgress(
+ frag: PartsLoadedData | FragLoadedData
+ ) {}
 
  protected _doFragLoad(
  frag: Fragment,
- details?: LevelDetails,
+ details: LevelDetails,
  targetBufferTime: number | null = null,
  progressCallback?: FragmentLoadProgressCallback
  ): Promise<PartsLoadedData | FragLoadedData | null> {
  if (!this.levels) {
  throw new Error('frag load aborted, missing levels');
  }
+
+ let keyLoadingPromise: Promise<KeyLoadedData | void> | null = null;
+ if (frag.encrypted && !frag.decryptdata?.key) {
+ this.log(
+ `Loading key for ${frag.sn} of [${details.startSN}-${details.endSN}], ${
+ this.logPrefix === '[stream-controller]' ? 'level' : 'track'
+ } ${frag.level}`
+ );
+ this.state = State.KEY_LOADING;
+ this.fragCurrent = frag;
+ keyLoadingPromise = this.keyLoader.load(frag).then((keyLoadedData) => {
+ if (keyLoadedData && !this.fragContextChanged(keyLoadedData.frag)) {
+ this.hls.trigger(Events.KEY_LOADED, keyLoadedData);
+ return keyLoadedData;
+ }
+ });
+ this.hls.trigger(Events.KEY_LOADING, { frag });
+ this.throwIfFragContextChanged('KEY_LOADING');
+ }
+
  targetBufferTime = Math.max(frag.start, targetBufferTime || 0);
  if (this.config.lowLatencyMode && details) {
  const partList = details.partList;
@@ -593,6 +596,26 @@ export default class BaseStreamController
  part: partList[partIndex],
  targetBufferTime,
  });
+ this.throwIfFragContextChanged('FRAG_LOADING parts');
+ if (keyLoadingPromise) {
+ return keyLoadingPromise
+ .then((keyLoadedData) => {
+ if (
+ !keyLoadedData ||
+ this.fragContextChanged(keyLoadedData?.frag)
+ ) {
+ return null;
+ }
+ return this.doFragPartsLoad(
+ frag,
+ partList,
+ partIndex,
+ progressCallback
+ );
+ })
+ .catch((error) => this.handleFragLoadError(error));
+ }
+
  return this.doFragPartsLoad(
  frag,
  partList,
@@ -622,10 +645,44 @@ export default class BaseStreamController
  }
  this.state = State.FRAG_LOADING;
  this.hls.trigger(Events.FRAG_LOADING, { frag, targetBufferTime });
+ this.throwIfFragContextChanged('FRAG_LOADING');
+
+ // Load key before streaming fragment data
+ const dataOnProgress = this.config.progressive;
+ if (dataOnProgress && keyLoadingPromise) {
+ return keyLoadingPromise
+ .then((keyLoadedData) => {
+ if (!keyLoadedData || this.fragContextChanged(keyLoadedData?.frag)) {
+ return null;
+ }
+ return this.fragmentLoader.load(frag, progressCallback);
+ })
+ .catch((error) => this.handleFragLoadError(error));
+ }
 
- return this.fragmentLoader
- .load(frag, progressCallback)
- .catch((error: LoadError) => this.handleFragLoadError(error));
+ // load unencrypted fragment data with progress event,
+ // or handle fragment result after key and fragment are finished loading
+ return Promise.all([
+ this.fragmentLoader.load(
+ frag,
+ dataOnProgress ? progressCallback : undefined
+ ),
+ keyLoadingPromise,
+ ])
+ .then(([fragLoadedData]) => {
+ if (!dataOnProgress && fragLoadedData && progressCallback) {
+ progressCallback(fragLoadedData);
+ }
+ return fragLoadedData;
+ })
+ .catch((error) => this.handleFragLoadError(error));
+ }
+
+ private throwIfFragContextChanged(context: string): void | never {
+ // exit if context changed during event loop
+ if (this.fragCurrent === null) {
+ throw new Error(`frag load aborted, context changed in ${context}`);
+ }
  }
 
  private doFragPartsLoad(

src/controller/stream-controller.ts

@@ -320,13 +320,7 @@ export default class StreamController
  frag = frag.initSegment;
  }
 
- // We want to load the key if we're dealing with an identity key, because we will decrypt
- // this content using the key we fetch. Other keys will be handled by the DRM CDM via EME.
- if (frag.decryptdata?.keyFormat === 'identity' && !frag.decryptdata?.key) {
- this.loadKey(frag, levelDetails);
- } else {
- this.loadFragment(frag, levelDetails, targetBufferTime);
- }
+ this.loadFragment(frag, levelDetails, targetBufferTime);
  }
 
  protected loadFragment(
@@ -339,12 +333,12 @@ export default class StreamController
  this.fragCurrent = frag;
  if (fragState === FragmentState.NOT_LOADED) {
  if (frag.sn === 'initSegment') {
- this._loadInitSegment(frag);
+ this._loadInitSegment(frag, levelDetails);
  } else if (this.bitrateTest) {
  this.log(
  `Fragment ${frag.sn} of level ${frag.level} is being downloaded to test bitrate and will not be buffered`
  );
- this._loadBitrateTestFrag(frag);
+ this._loadBitrateTestFrag(frag, levelDetails);
  } else {
  this.startFragRequested = true;
  super.loadFragment(frag, levelDetails, targetBufferTime);
@@ -465,8 +459,8 @@ export default class StreamController
  const fragCurrent = this.fragCurrent;
  this.fragCurrent = null;
  this.backtrackFragment = null;
- if (fragCurrent?.loader) {
- fragCurrent.loader.abort();
+ if (fragCurrent) {
+ fragCurrent.abortRequests();
  }
  switch (this.state) {
  case State.KEY_LOADING:
@@ -618,7 +612,7 @@ export default class StreamController
  if (fragCurrent.level !== data.level && fragCurrent.loader) {
  this.state = State.IDLE;
  this.backtrackFragment = null;
- fragCurrent.loader.abort();
+ fragCurrent.abortRequests();
  }
  }
 
@@ -740,9 +734,9 @@ export default class StreamController
  this.mediaBuffer = this.media;
  const fragCurrent = this.fragCurrent;
  // we need to refill audio buffer from main: cancel any frag loading to speed up audio switch
- if (fragCurrent?.loader) {
+ if (fragCurrent) {
  this.log('Switching to main audio track, cancel main fragment load');
- fragCurrent.loader.abort();
+ fragCurrent.abortRequests();
  }
  // destroy transmuxer to force init segment generation (following audio switch)
  this.resetTransmuxer();
@@ -1013,9 +1007,9 @@ export default class StreamController
  return audioCodec;
  }
 
- private _loadBitrateTestFrag(frag: Fragment) {
+ private _loadBitrateTestFrag(frag: Fragment, levelDetails: LevelDetails) {
  frag.bitrateTest = true;
- this._doFragLoad(frag).then((data) => {
+ this._doFragLoad(frag, levelDetails).then((data) => {
  const { hls } = this;
  if (!data || hls.nextLoadLevel || this.fragContextChanged(frag)) {
  return;