From 1e651dc0aea482d8606974eb1f2d863642ccb195 Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 14 Nov 2024 00:05:12 -0500 Subject: [PATCH 1/5] Stop skipping of all frames in MTS video --- server/src/services/media.service.spec.ts | 6 +++--- server/src/utils/media.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index df1a04dff8709..0f99df729b412 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -403,7 +403,7 @@ describe(MediaService.name, () => { '/original/path.ext', 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', expect.objectContaining({ - inputOptions: ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'], + inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'], outputOptions: [ '-fps_mode vfr', '-frames:v 1', @@ -438,7 +438,7 @@ describe(MediaService.name, () => { '/original/path.ext', 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', expect.objectContaining({ - inputOptions: ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'], + inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'], outputOptions: [ '-fps_mode vfr', '-frames:v 1', @@ -475,7 +475,7 @@ describe(MediaService.name, () => { '/original/path.ext', 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', expect.objectContaining({ - inputOptions: ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'], + inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'], outputOptions: [ '-fps_mode vfr', '-frames:v 1', diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index f61b472b7512d..597dabdb6accb 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -378,7 +378,7 @@ export class ThumbnailConfig extends BaseConfig { } getBaseInputOptions(): string[] { - return ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int']; + return ['-sws_flags accurate_rnd+full_chroma_int']; } getBaseOutputOptions() { From a2c95610db39d44fd081e05216a2ff72fde1620d Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 14 Nov 2024 01:20:19 -0500 Subject: [PATCH 2/5] Only skip flag for mts videos --- server/src/interfaces/media.interface.ts | 7 ++++++- server/src/services/media.service.spec.ts | 6 +++--- server/src/services/media.service.ts | 13 +++++++++---- server/src/utils/media.ts | 22 ++++++++++++++++------ 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/server/src/interfaces/media.interface.ts b/server/src/interfaces/media.interface.ts index fe11d17b5f415..468a6ad88d9ec 100644 --- a/server/src/interfaces/media.interface.ts +++ b/server/src/interfaces/media.interface.ts @@ -114,7 +114,12 @@ export interface ImageBuffer { } export interface VideoCodecSWConfig { - getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeCommand; + getCommand( + target: TranscodeTarget, + videoStream: VideoStreamInfo, + audioStream: AudioStreamInfo, + format?: VideoFormat, + ): TranscodeCommand; } export interface VideoCodecHWConfig extends VideoCodecSWConfig { diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 0f99df729b412..df1a04dff8709 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -403,7 +403,7 @@ describe(MediaService.name, () => { '/original/path.ext', 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', expect.objectContaining({ - inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'], + inputOptions: ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'], outputOptions: [ '-fps_mode vfr', '-frames:v 1', @@ -438,7 +438,7 @@ describe(MediaService.name, () => { '/original/path.ext', 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', expect.objectContaining({ - inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'], + inputOptions: ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'], outputOptions: [ '-fps_mode vfr', '-frames:v 1', @@ -475,7 +475,7 @@ describe(MediaService.name, () => { '/original/path.ext', 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', expect.objectContaining({ - inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'], + inputOptions: ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'], outputOptions: [ '-fps_mode vfr', '-frames:v 1', diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 0aa6bd03fb170..770e26b2436f1 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -239,7 +239,7 @@ export class MediaService extends BaseService { const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format); this.storageCore.ensureFolders(previewPath); - const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); + const { format, audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); const mainVideoStream = this.getMainStream(videoStreams); if (!mainVideoStream) { throw new Error(`No video streams found for asset ${asset.id}`); @@ -248,9 +248,14 @@ export class MediaService extends BaseService { const previewConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.preview.size.toString() }); const thumbnailConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() }); - - const previewOptions = previewConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream); - const thumbnailOptions = thumbnailConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream); + const previewOptions = previewConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream, format); + const thumbnailOptions = thumbnailConfig.getCommand( + TranscodeTarget.VIDEO, + mainVideoStream, + mainAudioStream, + format, + ); + this.logger.error(format.formatName); await this.mediaRepository.transcode(asset.originalPath, previewPath, previewOptions); await this.mediaRepository.transcode(asset.originalPath, thumbnailPath, thumbnailOptions); diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index 597dabdb6accb..afb3f99e2fe37 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -1,11 +1,12 @@ import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; -import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec } from 'src/enum'; +import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec, VideoContainer } from 'src/enum'; import { AudioStreamInfo, BitrateDistribution, TranscodeCommand, VideoCodecHWConfig, VideoCodecSWConfig, + VideoFormat, VideoStreamInfo, } from 'src/interfaces/media.interface'; @@ -77,9 +78,14 @@ export class BaseConfig implements VideoCodecSWConfig { return handler; } - getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { + getCommand( + target: TranscodeTarget, + videoStream: VideoStreamInfo, + audioStream?: AudioStreamInfo, + format?: VideoFormat, + ) { const options = { - inputOptions: this.getBaseInputOptions(videoStream), + inputOptions: this.getBaseInputOptions(videoStream, format), outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v verbose'], twoPass: this.eligibleForTwoPass(), progress: { frameCount: videoStream.frameCount, percentInterval: 5 }, @@ -101,7 +107,7 @@ export class BaseConfig implements VideoCodecSWConfig { } // eslint-disable-next-line @typescript-eslint/no-unused-vars - getBaseInputOptions(videoStream: VideoStreamInfo): string[] { + getBaseInputOptions(videoStream: VideoStreamInfo, format?: VideoFormat): string[] { return this.getInputThreadOptions(); } @@ -377,8 +383,12 @@ export class ThumbnailConfig extends BaseConfig { return new ThumbnailConfig(config); } - getBaseInputOptions(): string[] { - return ['-sws_flags accurate_rnd+full_chroma_int']; + getBaseInputOptions(videoStream: VideoStreamInfo, format?: VideoFormat): string[] { + if (format?.formatName === 'mpegts') { + return ['-sws_flags accurate_rnd+full_chroma_int']; + } else { + return ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int']; + } } getBaseOutputOptions() { From c66e4d49f24cb355846394b37d23564947e1feac Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 14 Nov 2024 01:30:59 -0500 Subject: [PATCH 3/5] Fix lint checks --- server/src/utils/media.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index afb3f99e2fe37..41fe3d27b9868 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -1,5 +1,5 @@ import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; -import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec, VideoContainer } from 'src/enum'; +import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec } from 'src/enum'; import { AudioStreamInfo, BitrateDistribution, @@ -384,11 +384,9 @@ export class ThumbnailConfig extends BaseConfig { } getBaseInputOptions(videoStream: VideoStreamInfo, format?: VideoFormat): string[] { - if (format?.formatName === 'mpegts') { - return ['-sws_flags accurate_rnd+full_chroma_int']; - } else { - return ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int']; - } + return format?.formatName === 'mpegts' + ? ['-sws_flags accurate_rnd+full_chroma_int'] + : ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int']; } getBaseOutputOptions() { From 3a75cff2411a0170459700b10fec2b0769eef6ee Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 14 Nov 2024 01:41:23 -0500 Subject: [PATCH 4/5] Adds test --- server/src/services/media.service.spec.ts | 16 ++++++++++++++++ server/test/fixtures/media.stub.ts | 7 +++++++ 2 files changed, 23 insertions(+) diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index df1a04dff8709..069376b8d358e 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -487,6 +487,22 @@ describe(MediaService.name, () => { }), ); }); + it('should not skip intra frames for MTS file', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamMTS); + assetMock.getById.mockResolvedValue(assetStub.video); + await sut.handleGenerateThumbnails({ id: assetStub.video.id }); + + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', + expect.objectContaining({ + inputOptions: ['-sws_flags accurate_rnd+full_chroma_int'], + outputOptions: expect.any(Array), + progress: expect.any(Object), + twoPass: false, + }), + ); + }); it('should use scaling divisible by 2 even when using quick sync', async () => { mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p); diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index 082959c2272c7..de11c23f0a961 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -95,6 +95,13 @@ export const probeStub = { ...probeStubDefault, videoStreams: [{ ...probeStubDefaultVideoStream[0], bitrate: 40_000_000 }], }), + videoStreamMTS: Object.freeze({ + ...probeStubDefault, + format: { + ...probeStubDefaultFormat, + formatName: 'mpegts', + }, + }), videoStreamHDR: Object.freeze({ ...probeStubDefault, videoStreams: [ From ef4ea719eed86ebe7e9cc47bed6ea3b96acaa07e Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 14 Nov 2024 01:55:40 -0500 Subject: [PATCH 5/5] Add comment for why flag is removed --- server/src/utils/media.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index 41fe3d27b9868..98d3c7fdbb208 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -384,6 +384,7 @@ export class ThumbnailConfig extends BaseConfig { } getBaseInputOptions(videoStream: VideoStreamInfo, format?: VideoFormat): string[] { + // skip_frame nointra skips all frames for some MPEG-TS files. Look at ffmpeg tickets 7950 and 7895 for more details. return format?.formatName === 'mpegts' ? ['-sws_flags accurate_rnd+full_chroma_int'] : ['-skip_frame nointra', '-sws_flags accurate_rnd+full_chroma_int'];