Skip to content

Commit

Permalink
Fix handling of negative composition timestamps (#4885)
Browse files Browse the repository at this point in the history 
* Fix handling of negative composition timestamps in ts->mp4 remuxing ("PTS < DTS" warnings)
Fixes #4878

* Only set trun version to 1 when track samples contain negative composition timestamps

* Only set trun version to 1 on video track trun boxes

* Do not reload partial fragments. They must be evicted from the tracker to reload.
(prevents loop loading on https://playertest.longtailvideo.com/adaptive/captions/playlist.m3u8 which now presents gaps in Safari)

* Fix for test stream with large DTS gap and bad composition timestamps
  • Loading branch information
@robwalch
robwalch authored Sep 9, 2022
1 parent 551f133 commit 6216ab9
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 62 deletions.
5 changes: 1 addition & 4 deletions src/controller/stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,10 +337,7 @@ export default class StreamController
// Check if fragment is not loaded
const fragState = this.fragmentTracker.getState(frag);
this.fragCurrent = frag;
if (
fragState === FragmentState.NOT_LOADED ||
fragState === FragmentState.PARTIAL
) {
if (fragState === FragmentState.NOT_LOADED) {
if (frag.sn === 'initSegment') {
this._loadInitSegment(frag);
} else if (this.bitrateTest) {
Expand Down
2 changes: 1 addition & 1 deletion src/remux/mp4-generator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1084,7 +1084,7 @@ class MP4 {
offset += 8 + arraylen;
array.set(
[
0x00, // version 0
track.type === 'video' ? 0x01 : 0x00, // version 1 for video with signed-int sample_composition_time_offset
0x00,
0x0f,
0x01, // flags
Expand Down
114 changes: 59 additions & 55 deletions src/remux/mp4-remuxer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,6 @@ export default class MP4Remuxer implements Remuxer {
let lastDTS;
let minPTS: number = Number.POSITIVE_INFINITY;
let maxPTS: number = Number.NEGATIVE_INFINITY;
let ptsDtsShift = 0;
let sortSamples = false;

// if parsed fragment is contiguous with last one, let's use last DTS value as reference
Expand All @@ -411,13 +410,6 @@ export default class MP4Remuxer implements Remuxer {
const sample = inputSamples[i];
sample.pts = normalizePts(sample.pts - initPTS, nextAvcDts);
sample.dts = normalizePts(sample.dts - initPTS, nextAvcDts);
if (sample.dts > sample.pts) {
const PTS_DTS_SHIFT_TOLERANCE_90KHZ = 90000 * 0.2;
ptsDtsShift = Math.max(
Math.min(ptsDtsShift, sample.pts - sample.dts),
-1 * PTS_DTS_SHIFT_TOLERANCE_90KHZ
);
}
if (sample.dts < inputSamples[i > 0 ? i - 1 : i].dts) {
sortSamples = true;
}
Expand All @@ -436,49 +428,13 @@ export default class MP4Remuxer implements Remuxer {
firstDTS = inputSamples[0].dts;
lastDTS = inputSamples[inputSamples.length - 1].dts;

// on Safari let's signal the same sample duration for all samples
// sample duration (as expected by trun MP4 boxes), should be the delta between sample DTS
// Sample duration (as expected by trun MP4 boxes), should be the delta between sample DTS
// set this constant duration as being the avg delta between consecutive DTS.
const inputDuration = lastDTS - firstDTS;
const averageSampleDuration = inputDuration
? Math.round(inputDuration / (nbSamples - 1))
: mp4SampleDuration || track.inputTimeScale / 30;

// handle broken streams with PTS < DTS, tolerance up 0.2 seconds
if (ptsDtsShift < 0) {
if (ptsDtsShift < averageSampleDuration * -2) {
// Fix for "CNN special report, with CC" in test-streams (including Safari browser)
// With large PTS < DTS errors such as this, we want to correct CTS while maintaining increasing DTS values
logger.warn(
`PTS < DTS detected in video samples, offsetting DTS from PTS by ${toMsFromMpegTsClock(
-averageSampleDuration,
true
)} ms`
);
let lastDts = ptsDtsShift;
for (let i = 0; i < nbSamples; i++) {
inputSamples[i].dts = lastDts = Math.max(
lastDts,
inputSamples[i].pts - averageSampleDuration
);
inputSamples[i].pts = Math.max(lastDts, inputSamples[i].pts);
}
} else {
// Fix for "Custom IV with bad PTS DTS" in test-streams
// With smaller PTS < DTS errors we can simply move all DTS back. This increases CTS without causing buffer gaps or decode errors in Safari
logger.warn(
`PTS < DTS detected in video samples, shifting DTS by ${toMsFromMpegTsClock(
ptsDtsShift,
true
)} ms to overcome this issue`
);
for (let i = 0; i < nbSamples; i++) {
inputSamples[i].dts = inputSamples[i].dts + ptsDtsShift;
}
}
firstDTS = inputSamples[0].dts;
}

// if fragment are contiguous, detect hole/overlapping between fragments
if (contiguous) {
// check timestamp continuity across consecutive fragments (this is to remove inter-fragment gap/hole)
Expand Down Expand Up @@ -536,11 +492,9 @@ export default class MP4Remuxer implements Remuxer {
nbNalu += nbUnits;
sample.length = sampleLen;

// normalize PTS/DTS
// ensure sample monotonic DTS
sample.dts = Math.max(sample.dts, firstDTS);
// ensure that computed value is greater or equal than sample DTS
sample.pts = Math.max(sample.pts, sample.dts, 0);

minPTS = Math.min(sample.pts, minPTS);
maxPTS = Math.max(sample.pts, maxPTS);
}
Expand All @@ -567,6 +521,10 @@ export default class MP4Remuxer implements Remuxer {
mdat.set(MP4.types.mdat, 4);

let stretchedLastFrame = false;
let minDtsDelta = Number.POSITIVE_INFINITY;
let minPtsDelta = Number.POSITIVE_INFINITY;
let maxDtsDelta = Number.NEGATIVE_INFINITY;
let maxPtsDelta = Number.NEGATIVE_INFINITY;
for (let i = 0; i < nbSamples; i++) {
const avcSample = inputSamples[i];
const avcSampleUnits = avcSample.units;
Expand All @@ -584,14 +542,20 @@ export default class MP4Remuxer implements Remuxer {
}

// expected sample duration is the Decoding Timestamp diff of consecutive samples
let ptsDelta;
if (i < nbSamples - 1) {
mp4SampleDuration = inputSamples[i + 1].dts - avcSample.dts;
ptsDelta = inputSamples[i + 1].pts - avcSample.pts;
} else {
const config = this.config;
const lastFrameDuration =
i > 0
? avcSample.dts - inputSamples[i - 1].dts
: averageSampleDuration;
ptsDelta =
i > 0
? avcSample.pts - inputSamples[i - 1].pts
: averageSampleDuration;
if (config.stretchShortVideoTrack && this.nextAudioPts !== null) {
// In some cases, a segment's audio track duration may exceed the video track duration.
// Since we've already remuxed audio, and we know how long the audio track is, we look to
Expand Down Expand Up @@ -627,6 +591,10 @@ export default class MP4Remuxer implements Remuxer {
}
}
const compositionTimeOffset = Math.round(avcSample.pts - avcSample.dts);
minDtsDelta = Math.min(minDtsDelta, mp4SampleDuration);
maxDtsDelta = Math.max(maxDtsDelta, mp4SampleDuration);
minPtsDelta = Math.min(minPtsDelta, ptsDelta);
maxPtsDelta = Math.max(maxPtsDelta, ptsDelta);

outputSamples.push(
new Mp4Sample(
Expand All @@ -638,12 +606,43 @@ export default class MP4Remuxer implements Remuxer {
);
}

if (outputSamples.length && chromeVersion && chromeVersion < 70) {
// Chrome workaround, mark first sample as being a Random Access Point (keyframe) to avoid sourcebuffer append issue
// https://code.google.com/p/chromium/issues/detail?id=229412
const flags = outputSamples[0].flags;
flags.dependsOn = 2;
flags.isNonSync = 0;
if (outputSamples.length) {
if (chromeVersion) {
if (chromeVersion < 70) {
// Chrome workaround, mark first sample as being a Random Access Point (keyframe) to avoid sourcebuffer append issue
// https://code.google.com/p/chromium/issues/detail?id=229412
const flags = outputSamples[0].flags;
flags.dependsOn = 2;
flags.isNonSync = 0;
}
} else if (safariWebkitVersion) {
// Fix for "CNN special report, with CC" in test-streams (Safari browser only)
// Ignore DTS when frame durations are irregular. Safari MSE does not handle this leading to gaps.
if (
maxPtsDelta - minPtsDelta < maxDtsDelta - minDtsDelta &&
averageSampleDuration / maxDtsDelta < 0.025 &&
outputSamples[0].cts === 0
) {
logger.warn(
'Found irregular gaps in sample duration. Using PTS instead of DTS to determine MP4 sample duration.'
);
let dts = firstDTS;
for (let i = 0, len = outputSamples.length; i < len; i++) {
const nextDts = dts + outputSamples[i].duration;
const pts = dts + outputSamples[i].cts;
if (i < len - 1) {
const nextPts = nextDts + outputSamples[i + 1].cts;
outputSamples[i].duration = nextPts - pts;
} else {
outputSamples[i].duration = i
? outputSamples[i - 1].duration
: averageSampleDuration;
}
outputSamples[i].cts = 0;
dts = nextDts;
}
}
}
}

console.assert(
Expand Down Expand Up @@ -1096,7 +1095,12 @@ class Mp4Sample {
public cts: number;
public flags: Mp4SampleFlags;

constructor(isKeyframe: boolean, duration, size, cts) {
constructor(
isKeyframe: boolean,
duration: number,
size: number,
cts: number
) {
this.duration = duration;
this.size = size;
this.cts = cts;
Expand Down
4 changes: 2 additions & 2 deletions tests/unit/controller/stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,10 +325,10 @@ describe('StreamController', function () {
assertLoadingState(frag);
});

it('should load a partial fragment', function () {
it('should not load a partial fragment', function () {
fragStateStub(FragmentState.PARTIAL);
streamController['loadFragment'](frag, levelDetails, 0);
assertLoadingState(frag);
assertNotLoadingState();
});

it('should not load a fragment which has completely & successfully loaded', function () {
Expand Down

0 comments on commit 6216ab9

Please sign in to comment.