fix: audio segment on incorrect timeline (#1530)

src/playlist-controller.js

@@ -929,6 +929,28 @@ export class PlaylistController extends videojs.EventTarget {
  this.onEndOfStream();
  });
 
+ // In DASH, there is the possibility of the video segment and the audio segment
+ // at a current time to be on different timelines. When this occurs, the player
+ // forwards playback to a point where these two segment types are back on the same
+ // timeline. This time will be just after the end of the audio segment that is on
+ // a previous timeline.
+ if (this.sourceType_ === 'dash') {
+ this.timelineChangeController_.on('audioTimelineBehind', () => {
+ const segmentInfo = this.audioSegmentLoader_.pendingSegment_;
+
+ if (!segmentInfo || !segmentInfo.segment || !segmentInfo.segment.syncInfo) {
+ return;
+ }
+
+ // Update the current time to just after the faulty audio segment.
+ // This moves playback to a spot where both audio and video segments
+ // are on the same timeline.
+ const newTime = segmentInfo.segment.syncInfo.end + 0.01;
+
+ this.tech_.setCurrentTime(newTime);
+ });
+ }
+
  this.mainSegmentLoader_.on('earlyabort', (event) => {
  // never try to early abort with the new ABR algorithm
  if (this.bufferBasedABR) {

src/segment-loader.js

@@ -423,15 +423,68 @@ export const shouldFixBadTimelineChanges = (timelineChangeController) => {
  return false;
 };
 
+/**
+ * Fixes certain bad timeline scenarios by resetting the loader.
+ *
+ * @param {SegmentLoader} segmentLoader
+ */
 export const fixBadTimelineChange = (segmentLoader) => {
  if (!segmentLoader) {
  return;
  }
+
  segmentLoader.pause();
  segmentLoader.resetEverything();
  segmentLoader.load();
 };
 
+/**
+ * Check if the pending audio timeline change is behind the
+ * pending main timeline change.
+ *
+ * @param {SegmentLoader} segmentLoader
+ * @return {boolean}
+ */
+const isAudioTimelineBehind = (segmentLoader) => {
+ const pendingAudioTimelineChange = segmentLoader.timelineChangeController_.pendingTimelineChange({ type: 'audio' });
+ const pendingMainTimelineChange = segmentLoader.timelineChangeController_.pendingTimelineChange({ type: 'main' });
+ const hasPendingTimelineChanges = pendingAudioTimelineChange && pendingMainTimelineChange;
+
+ return hasPendingTimelineChanges && pendingAudioTimelineChange.to < pendingMainTimelineChange.to;
+};
+
+/**
+ * A method to check if the player is waiting for a timeline change, and fixes
+ * certain scenarios where the timelines need to be updated.
+ *
+ * @param {SegmentLoader} segmentLoader
+ */
+const checkAndFixTimelines = (segmentLoader) => {
+ const segmentInfo = segmentLoader.pendingSegment_;
+
+ if (!segmentInfo) {
+ return;
+ }
+
+ const waitingForTimelineChange = shouldWaitForTimelineChange({
+ timelineChangeController: segmentLoader.timelineChangeController_,
+ currentTimeline: segmentLoader.currentTimeline_,
+ segmentTimeline: segmentInfo.timeline,
+ loaderType: segmentLoader.loaderType_,
+ audioDisabled: segmentLoader.audioDisabled_
+ });
+
+ if (waitingForTimelineChange && shouldFixBadTimelineChanges(segmentLoader.timelineChangeController_)) {
+ // Audio being behind should only happen on DASH sources.
+ if (segmentLoader.sourceType_ === 'dash' && isAudioTimelineBehind(segmentLoader)) {
+ segmentLoader.timelineChangeController_.trigger('audioTimelineBehind');
+ return;
+ }
+
+ fixBadTimelineChange(segmentLoader);
+ }
+};
+
 export const mediaDuration = (timingInfos) => {
  let maxDuration = 0;
 
@@ -698,6 +751,8 @@ export default class SegmentLoader extends videojs.EventTarget {
  this.sourceUpdater_.on('ready', () => {
  if (this.hasEnoughInfoToAppend_()) {
  this.processCallQueue_();
+ } else {
+ checkAndFixTimelines(this);
  }
  });
 
@@ -712,6 +767,8 @@ export default class SegmentLoader extends videojs.EventTarget {
  this.timelineChangeController_.on('pendingtimelinechange', () => {
  if (this.hasEnoughInfoToAppend_()) {
  this.processCallQueue_();
+ } else {
+ checkAndFixTimelines(this);
  }
  });
  }
@@ -723,9 +780,13 @@ export default class SegmentLoader extends videojs.EventTarget {
  this.trigger({type: 'timelinechange', ...metadata });
  if (this.hasEnoughInfoToLoad_()) {
  this.processLoadQueue_();
+ } else {
+ checkAndFixTimelines(this);
  }
  if (this.hasEnoughInfoToAppend_()) {
  this.processCallQueue_();
+ } else {
+ checkAndFixTimelines(this);
  }
  });
  }
@@ -1961,6 +2022,8 @@ Fetch At Buffer: ${this.fetchAtBuffer_}
  // check if any calls were waiting on the track info
  if (this.hasEnoughInfoToAppend_()) {
  this.processCallQueue_();
+ } else {
+ checkAndFixTimelines(this);
  }
  }
 
@@ -1981,6 +2044,8 @@ Fetch At Buffer: ${this.fetchAtBuffer_}
  // check if any calls were waiting on the timing info
  if (this.hasEnoughInfoToAppend_()) {
  this.processCallQueue_();
+ } else {
+ checkAndFixTimelines(this);
  }
  }
 
@@ -2156,9 +2221,6 @@ Fetch At Buffer: ${this.fetchAtBuffer_}
  audioDisabled: this.audioDisabled_
  })
  ) {
- if (shouldFixBadTimelineChanges(this.timelineChangeController_)) {
- fixBadTimelineChange(this);
- }
  return false;
  }
 
@@ -2219,9 +2281,6 @@ Fetch At Buffer: ${this.fetchAtBuffer_}
  audioDisabled: this.audioDisabled_
  })
  ) {
- if (shouldFixBadTimelineChanges(this.timelineChangeController_)) {
- fixBadTimelineChange(this);
- }
  return false;
  }
 
@@ -2238,6 +2297,8 @@ Fetch At Buffer: ${this.fetchAtBuffer_}
  // If there's anything in the call queue, then this data came later and should be
  // executed after the calls currently queued.
  if (this.callQueue_.length || !this.hasEnoughInfoToAppend_()) {
+ checkAndFixTimelines(this);
+
  this.callQueue_.push(this.handleData_.bind(this, simpleSegment, result));
  return;
  }
@@ -2627,6 +2688,8 @@ Fetch At Buffer: ${this.fetchAtBuffer_}
  }
 
  if (!this.hasEnoughInfoToLoad_()) {
+ checkAndFixTimelines(this);
+
  this.loadQueue_.push(() => {
  // regenerate the audioAppendStart, timestampOffset, etc as they
  // may have changed since this function was added to the queue.

test/playlist-controller.test.js

@@ -2647,6 +2647,52 @@ QUnit.test(
  }
 );
 
+QUnit.test(
+ 'setCurrentTime is not called on audioTimelineBehind when there is no pending segment',
+ function(assert) {
+ const options = {
+ src: 'test',
+ tech: this.player.tech_,
+ sourceType: 'dash',
+ player_: this.player
+ };
+ const pc = new PlaylistController(options);
+ const tech = this.player.tech_;
+ const setCurrentTimeSpy = sinon.spy(tech, 'setCurrentTime');
+
+ pc.timelineChangeController_.trigger('audioTimelineBehind');
+
+ assert.notOk(setCurrentTimeSpy.called, 'setCurrentTime not called');
+ }
+);
+
+QUnit.test(
+ 'setCurrentTime to after audio segment when audioTimelineBehind is triggered',
+ function(assert) {
+ const options = {
+ src: 'test',
+ tech: this.player.tech_,
+ sourceType: 'dash',
+ player_: this.player
+ };
+ const pc = new PlaylistController(options);
+ const tech = this.player.tech_;
+ const setCurrentTimeSpy = sinon.spy(tech, 'setCurrentTime');
+
+ pc.audioSegmentLoader_.pendingSegment_ = {
+ segment: {
+ syncInfo: {
+ end: 10
+ }
+ }
+ };
+
+ pc.timelineChangeController_.trigger('audioTimelineBehind');
+
+ assert.ok(setCurrentTimeSpy.calledWith(10.01), 'sets current time to just after the end of the audio segment');
+ }
+);
+
 QUnit.test('calls to update cues on new media', function(assert) {
  const origVhsOptions = videojs.options.vhs;
 

test/segment-loader.test.js

@@ -1663,7 +1663,7 @@ QUnit.module('SegmentLoader', function(hooks) {
  });
  });
 
- QUnit.test('hasEnoughInfoToLoad_ calls fixBadTimelineChange', function(assert) {
+ QUnit.test('fixBadTimelineChange on load', function(assert) {
  loader.dispose();
  loader = new SegmentLoader(LoaderCommonSettings.call(this, {
  loaderType: 'audio'
@@ -1722,7 +1722,7 @@ QUnit.module('SegmentLoader', function(hooks) {
  });
  });
 
- QUnit.test('hasEnoughInfoToAppend_ calls fixBadTimelineChange', function(assert) {
+ QUnit.test('fixBadTimelineChange on append', function(assert) {
  loader.dispose();
  loader = new SegmentLoader(LoaderCommonSettings.call(this, {
  loaderType: 'main'
@@ -1789,6 +1789,48 @@ QUnit.module('SegmentLoader', function(hooks) {
  });
  });
 
+ QUnit.test('triggers event when DASH audio timeline is behind main', function(assert) {
+ loader.dispose();
+ loader = new SegmentLoader(LoaderCommonSettings.call(this, {
+ loaderType: 'audio',
+ sourceType: 'dash'
+ }), {});
+
+ loader.timelineChangeController_.pendingTimelineChange = ({ type }) => {
+ if (type === 'audio') {
+ return {
+ from: 0,
+ to: 5
+ };
+ } else if (type === 'main') {
+ return {
+ from: 0,
+ to: 10
+ };
+ }
+ };
+
+ const triggerSpy = sinon.spy(loader.timelineChangeController_, 'trigger');
+
+ const playlist = playlistWithDuration(20);
+
+ playlist.discontinuityStarts = [1];
+ loader.getCurrentMediaInfo_ = () => {
+ return {
+ hasVideo: true,
+ hasAudio: true,
+ isMuxed: true
+ };
+ };
+
+ return this.setupMediaSource(loader.mediaSource_, loader.sourceUpdater_).then(() => {
+ loader.playlist(playlist);
+ loader.load();
+ this.clock.tick(1);
+ assert.ok(triggerSpy.calledWith('audioTimelineBehind'), 'audio timeline behind event is triggered');
+ });
+ });
+
  QUnit.test('audio loader does not wait to request segment even if timestamp offset is nonzero', function(assert) {
  loader.dispose();
  loader = new SegmentLoader(LoaderCommonSettings.call(this, {