From 756e54c69ffa7beb7a886c5fb79df3f78ff9d3f9 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Tue, 22 Oct 2024 18:44:59 -0400 Subject: [PATCH 1/7] use tonemapx, check input pixel format --- server/src/interfaces/media.interface.ts | 1 + server/src/repositories/media.repository.ts | 1 + server/src/services/media.service.spec.ts | 109 +++++++++++--- server/src/services/media.service.ts | 2 +- server/src/utils/media.ts | 156 +++++++------------- server/test/fixtures/media.stub.ts | 25 ++++ 6 files changed, 169 insertions(+), 125 deletions(-) diff --git a/server/src/interfaces/media.interface.ts b/server/src/interfaces/media.interface.ts index 2bc8ccde36d8b..d8d7395ea7b0f 100644 --- a/server/src/interfaces/media.interface.ts +++ b/server/src/interfaces/media.interface.ts @@ -59,6 +59,7 @@ export interface VideoStreamInfo { frameCount: number; isHDR: boolean; bitrate: number; + pixelFormat: string; } export interface AudioStreamInfo { diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 7e1ca84993fed..d6b0ba47e206b 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -126,6 +126,7 @@ export class MediaRepository implements IMediaRepository { rotation: this.parseInt(stream.rotation), isHDR: stream.color_transfer === 'smpte2084' || stream.color_transfer === 'arib-std-b67', bitrate: this.parseInt(stream.bit_rate), + pixelFormat: stream.pix_fmt || 'yuv420p', })), audioStreams: results.streams .filter((stream) => stream.codec_type === 'audio') diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index afd21ee9e99e6..b6ec01d9054de 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -410,7 +410,7 @@ describe(MediaService.name, () => { '-frames:v 1', '-update 1', '-v verbose', - String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_color_matrix=bt601:out_range=pc,format=yuv420p`, + String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc`, ], twoPass: false, }), @@ -445,7 +445,7 @@ describe(MediaService.name, () => { '-frames:v 1', '-update 1', '-v verbose', - String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=601:m=470bg:range=pc,format=yuv420p`, + String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`, ], twoPass: false, }), @@ -482,7 +482,7 @@ describe(MediaService.name, () => { '-frames:v 1', '-update 1', '-v verbose', - String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=601:m=470bg:range=pc,format=yuv420p`, + String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`, ], twoPass: false, }), @@ -1328,7 +1328,7 @@ describe(MediaService.name, () => { '-map 0:0', '-map 0:1', '-v verbose', - '-vf scale=-2:720,format=yuv420p', + '-vf scale=-2:720', '-preset 12', '-crf 23', ]), @@ -1454,7 +1454,7 @@ describe(MediaService.name, () => { '-map 0:1', '-g 256', '-v verbose', - '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720', + '-vf hwupload_cuda,scale_cuda=-2:720:format=nv12', '-preset p1', '-cq:v 23', ]), @@ -1586,7 +1586,7 @@ describe(MediaService.name, () => { inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']), outputOptions: expect.arrayContaining([ expect.stringContaining( - 'tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709:format=nv12', + 'tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:tonemap_mode=lum:transfer=bt709:peak=100:format=nv12', ), ]), twoPass: false, @@ -1594,6 +1594,24 @@ describe(MediaService.name, () => { ); }); + it('should set format to nv12 for nvenc if input is not yuv420p', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit); + systemMock.get.mockResolvedValue({ + ffmpeg: { accel: TranscodeHWAccel.NVENC, accelDecode: true }, + }); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + expect.objectContaining({ + inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']), + outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]), + twoPass: false, + }), + ); + }); + it('should set options for qsv', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); @@ -1616,7 +1634,7 @@ describe(MediaService.name, () => { '-refs 5', '-g 256', '-v verbose', - '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq', + '-vf hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq:format=nv12', '-preset 7', '-global_quality:v 23', '-maxrate 10000k', @@ -1748,7 +1766,7 @@ describe(MediaService.name, () => { ]), outputOptions: expect.arrayContaining([ expect.stringContaining( - 'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=qsv:reverse=1,format=qsv', + 'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:transfer=bt709:range=pc:tonemap=hable:tonemap_mode=lum:peak=100,hwmap=derive_device=qsv:reverse=1,format=qsv', ), ]), twoPass: false, @@ -1776,6 +1794,32 @@ describe(MediaService.name, () => { ); }); + it('should set format to nv12 for qsv if input is not yuv420p', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit); + systemMock.get.mockResolvedValue({ + ffmpeg: { accel: TranscodeHWAccel.QSV, accelDecode: true }, + }); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + + await sut.handleVideoConversion({ id: assetStub.video.id }); + + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + expect.objectContaining({ + inputOptions: expect.arrayContaining([ + '-hwaccel qsv', + '-hwaccel_output_format qsv', + '-async_depth 4', + '-threads 1', + ]), + outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]), + twoPass: false, + }), + ); + }); + it('should set options for vaapi', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); @@ -1799,7 +1843,7 @@ describe(MediaService.name, () => { '-map 0:1', '-g 256', '-v verbose', - '-vf format=nv12,hwupload,scale_vaapi=-2:720:mode=hq:out_range=pc', + '-vf hwupload=extra_hw_frames=64,scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12', '-compression_level 7', '-rc_mode 1', ]), @@ -1970,7 +2014,7 @@ describe(MediaService.name, () => { ); }); - it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => { + it('should use hardware tone-mapping for vaapi if hardware decoding is enabled and should tone map', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); systemMock.get.mockResolvedValue({ @@ -1987,7 +2031,7 @@ describe(MediaService.name, () => { inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']), outputOptions: expect.arrayContaining([ expect.stringContaining( - 'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1,format=vaapi', + 'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:transfer=bt709:range=pc:tonemap=hable:tonemap_mode=lum:peak=100,hwmap=derive_device=vaapi:reverse=1,format=vaapi', ), ]), twoPass: false, @@ -1995,6 +2039,27 @@ describe(MediaService.name, () => { ); }); + it('should set format to nv12 for vaapi if input is not yuv420p', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit); + systemMock.get.mockResolvedValue({ + ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true }, + }); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + + await sut.handleVideoConversion({ id: assetStub.video.id }); + + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + expect.objectContaining({ + inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']), + outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]), + twoPass: false, + }), + ); + }); + it('should use preferred device for vaapi when hardware decoding', async () => { storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); @@ -2140,7 +2205,7 @@ describe(MediaService.name, () => { inputOptions: expect.arrayContaining(['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']), outputOptions: expect.arrayContaining([ expect.stringContaining( - 'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime', + 'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0:tonemap_mode=lum:peak=100,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime', ), ]), twoPass: false, @@ -2164,7 +2229,7 @@ describe(MediaService.name, () => { inputOptions: [], outputOptions: expect.arrayContaining([ expect.stringContaining( - 'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p', + 'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p', ), ]), twoPass: false, @@ -2188,7 +2253,7 @@ describe(MediaService.name, () => { inputOptions: [], outputOptions: expect.arrayContaining([ expect.stringContaining( - 'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p', + 'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p', ), ]), twoPass: false, @@ -2209,7 +2274,7 @@ describe(MediaService.name, () => { outputOptions: expect.arrayContaining([ '-c:v h264', '-c:a copy', - '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p', + '-vf tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p', ]), twoPass: false, }), @@ -2229,16 +2294,16 @@ describe(MediaService.name, () => { outputOptions: expect.arrayContaining([ '-c:v h264', '-c:a copy', - '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p', + '-vf tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p', ]), twoPass: false, }), ); }); - it('should set npl to 250 for reinhard and mobius tone-mapping algorithms', async () => { - mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); - systemMock.get.mockResolvedValue({ ffmpeg: { tonemap: ToneMapping.MOBIUS } }); + it('should transcode when policy is required and video is not yuv420p', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit); + systemMock.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.REQUIRED } }); assetMock.getByIds.mockResolvedValue([assetStub.video]); await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( @@ -2246,11 +2311,7 @@ describe(MediaService.name, () => { 'upload/encoded-video/user-id/as/se/asset-id.mp4', expect.objectContaining({ inputOptions: expect.any(Array), - outputOptions: expect.arrayContaining([ - '-c:v h264', - '-c:a copy', - '-vf zscale=t=linear:npl=250,tonemap=mobius:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p', - ]), + outputOptions: expect.arrayContaining(['-c:v h264', '-c:a copy', '-vf format=yuv420p']), twoPass: false, }), ); diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 8393f5dc76282..e6c7ebb1e124f 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -407,7 +407,7 @@ export class MediaService extends BaseService { const isLargerThanTargetBitrate = stream.bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate); const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(stream.codecName as VideoCodec); - const isRequired = !isTargetVideoCodec || stream.isHDR; + const isRequired = !isTargetVideoCodec || stream.pixelFormat !== 'yuv420p'; switch (ffmpegConfig.transcode) { case TranscodePolicy.DISABLED: { diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index 03d57296d83b8..06726943a79a1 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -149,7 +149,11 @@ export class BaseConfig implements VideoCodecSWConfig { options.push(`scale=${this.getScaling(videoStream)}`); } - options.push(...this.getToneMapping(videoStream), 'format=yuv420p'); + options.push(...this.getToneMapping(videoStream)); + if (options.length === 0 && videoStream.pixelFormat !== 'yuv420p') { + options.push(`format=yuv420p`); + } + return options; } @@ -271,33 +275,20 @@ export class BaseConfig implements VideoCodecSWConfig { getColors() { return { - primaries: '709', - transfer: '709', - matrix: '709', + primaries: 'bt709', + transfer: 'bt709', + matrix: 'bt709', }; } - getNPL() { - if (this.config.npl <= 0) { - // since hable already outputs a darker image, we use a lower npl value for it - return this.config.tonemap === ToneMapping.HABLE ? 100 : 250; - } else { - return this.config.npl; - } - } - getToneMapping(videoStream: VideoStreamInfo) { if (!this.shouldToneMap(videoStream)) { return []; } - const colors = this.getColors(); - - return [ - `zscale=t=linear:npl=${this.getNPL()}`, - `tonemap=${this.config.tonemap}:desat=0`, - `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`, - ]; + const { primaries, transfer, matrix } = this.getColors(); + const options = `tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=yuv420p`; + return [options]; } getAudioCodec(): string { @@ -395,19 +386,14 @@ export class ThumbnailConfig extends BaseConfig { } getFilterOptions(videoStream: VideoStreamInfo): string[] { - const options = [ + return [ 'fps=12:eof_action=pass:round=down', 'thumbnail=12', String.raw`select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20)`, 'trim=end_frame=2', 'reverse', + ...super.getFilterOptions(videoStream), ]; - if (this.shouldScale(videoStream)) { - options.push(`scale=${this.getScaling(videoStream)}`); - } - - options.push(...this.getToneMapping(videoStream), 'format=yuv420p'); - return options; } getPresetOptions() { @@ -423,19 +409,7 @@ export class ThumbnailConfig extends BaseConfig { } getScaling(videoStream: VideoStreamInfo) { - let options = super.getScaling(videoStream) + ':flags=lanczos+accurate_rnd+full_chroma_int'; - if (!this.shouldToneMap(videoStream)) { - options += ':out_color_matrix=bt601:out_range=pc'; - } - return options; - } - - getColors() { - return { - primaries: '709', - transfer: '601', - matrix: '470bg', - }; + return super.getScaling(videoStream) + ':flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc'; } } @@ -559,9 +533,9 @@ export class NvencSwDecodeConfig extends BaseHWConfig { getFilterOptions(videoStream: VideoStreamInfo) { const options = this.getToneMapping(videoStream); - options.push('format=nv12', 'hwupload_cuda'); + options.push('hwupload_cuda'); if (this.shouldScale(videoStream)) { - options.push(`scale_cuda=${this.getScaling(videoStream)}`); + options.push(`scale_cuda=${this.getScaling(videoStream)}:format=nv12`); } return options; @@ -622,6 +596,8 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig { options.push(...this.getToneMapping(videoStream)); if (options.length > 0) { options[options.length - 1] += ':format=nv12'; + } else if (videoStream.pixelFormat !== 'yuv420p') { + options.push('format=nv12'); } return options; } @@ -631,14 +607,16 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig { return []; } - const colors = this.getColors(); + const { matrix, primaries, transfer } = this.getColors(); const tonemapOptions = [ 'desat=0', - `matrix=${colors.matrix}`, - `primaries=${colors.primaries}`, + `matrix=${matrix}`, + `primaries=${primaries}`, 'range=pc', `tonemap=${this.config.tonemap}`, - `transfer=${colors.transfer}`, + 'tonemap_mode=lum', + `transfer=${transfer}`, + 'peak=100', ]; return [`tonemap_cuda=${tonemapOptions.join(':')}`]; @@ -651,14 +629,6 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig { getOutputThreadOptions() { return []; } - - getColors() { - return { - primaries: 'bt709', - transfer: 'bt709', - matrix: 'bt709', - }; - } } export class QsvSwDecodeConfig extends BaseHWConfig { @@ -687,9 +657,9 @@ export class QsvSwDecodeConfig extends BaseHWConfig { getFilterOptions(videoStream: VideoStreamInfo) { const options = this.getToneMapping(videoStream); - options.push('format=nv12', 'hwupload=extra_hw_frames=64'); + options.push('hwupload=extra_hw_frames=64'); if (this.shouldScale(videoStream)) { - options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq`); + options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq:format=nv12`); } return options; } @@ -764,15 +734,18 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig { getFilterOptions(videoStream: VideoStreamInfo) { const options = []; - if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) { + const tonemapOptions = this.getToneMapping(videoStream); + if (this.shouldScale(videoStream) || tonemapOptions.length === 0) { let scaling = `scale_qsv=${this.getScaling(videoStream)}:async_depth=4:mode=hq`; - if (!this.shouldToneMap(videoStream)) { + if (tonemapOptions.length === 0) { scaling += ':format=nv12'; } options.push(scaling); } - - options.push(...this.getToneMapping(videoStream)); + options.push(...tonemapOptions); + if (options.length === 0 && videoStream.pixelFormat !== 'yuv420p') { + options.push('format=nv12'); + } return options; } @@ -781,15 +754,17 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig { return []; } - const colors = this.getColors(); + const { matrix, primaries, transfer } = this.getColors(); const tonemapOptions = [ 'desat=0', 'format=nv12', - `matrix=${colors.matrix}`, - `primaries=${colors.primaries}`, + `matrix=${matrix}`, + `primaries=${primaries}`, + `transfer=${transfer}`, 'range=pc', `tonemap=${this.config.tonemap}`, - `transfer=${colors.transfer}`, + 'tonemap_mode=lum', + 'peak=100', ]; return [ @@ -802,14 +777,6 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig { getInputThreadOptions() { return [`-threads 1`]; } - - getColors() { - return { - primaries: 'bt709', - transfer: 'bt709', - matrix: 'bt709', - }; - } } export class VaapiSwDecodeConfig extends BaseHWConfig { @@ -828,9 +795,9 @@ export class VaapiSwDecodeConfig extends BaseHWConfig { getFilterOptions(videoStream: VideoStreamInfo) { const options = this.getToneMapping(videoStream); - options.push('format=nv12', 'hwupload'); + options.push('hwupload=extra_hw_frames=64'); if (this.shouldScale(videoStream)) { - options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`); + options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc:format=nv12`); } return options; @@ -901,15 +868,18 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig { getFilterOptions(videoStream: VideoStreamInfo) { const options = []; - if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) { + const tonemapOptions = this.getToneMapping(videoStream); + if (this.shouldScale(videoStream) || tonemapOptions.length === 0) { let scaling = `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`; - if (!this.shouldToneMap(videoStream)) { + if (tonemapOptions.length === 0) { scaling += ':format=nv12'; } options.push(scaling); } - - options.push(...this.getToneMapping(videoStream)); + options.push(...tonemapOptions); + if (options.length === 0 && videoStream.pixelFormat !== 'yuv420p') { + options.push('format=nv12'); + } return options; } @@ -918,15 +888,17 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig { return []; } - const colors = this.getColors(); + const { matrix, primaries, transfer } = this.getColors(); const tonemapOptions = [ 'desat=0', 'format=nv12', - `matrix=${colors.matrix}`, - `primaries=${colors.primaries}`, + `matrix=${matrix}`, + `primaries=${primaries}`, + `transfer=${transfer}`, 'range=pc', `tonemap=${this.config.tonemap}`, - `transfer=${colors.transfer}`, + 'tonemap_mode=lum', + 'peak=100', ]; return [ @@ -939,14 +911,6 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig { getInputThreadOptions() { return [`-threads 1`]; } - - getColors() { - return { - primaries: 'bt709', - transfer: 'bt709', - matrix: 'bt709', - }; - } } export class RkmppSwDecodeConfig extends BaseHWConfig { @@ -1014,11 +978,11 @@ export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig { getFilterOptions(videoStream: VideoStreamInfo) { if (this.shouldToneMap(videoStream)) { - const colors = this.getColors(); + const { primaries, transfer, matrix } = this.getColors(); return [ `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`, 'hwmap=derive_device=opencl:mode=read', - `tonemap_opencl=format=nv12:r=pc:p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:tonemap=${this.config.tonemap}:desat=0`, + `tonemap_opencl=format=nv12:r=pc:p=${primaries}:t=${transfer}:m=${matrix}:tonemap=${this.config.tonemap}:desat=0:tonemap_mode=lum:peak=100`, 'hwmap=derive_device=rkmpp:mode=write:reverse=1', 'format=drm_prime', ]; @@ -1027,12 +991,4 @@ export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig { } return []; } - - getColors() { - return { - primaries: 'bt709', - transfer: 'bt709', - matrix: 'bt709', - }; - } } diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index cdcdfd4d5e5d9..082959c2272c7 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -17,6 +17,7 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [ rotation: 0, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, ]; @@ -43,6 +44,7 @@ export const probeStub = { rotation: 0, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, { index: 1, @@ -53,6 +55,7 @@ export const probeStub = { rotation: 0, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, ], }), @@ -68,6 +71,7 @@ export const probeStub = { rotation: 0, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, ], }), @@ -83,6 +87,7 @@ export const probeStub = { rotation: 0, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, ], }), @@ -102,6 +107,23 @@ export const probeStub = { rotation: 0, isHDR: true, bitrate: 0, + pixelFormat: 'yuv420p10le', + }, + ], + }), + videoStream10Bit: Object.freeze({ + ...probeStubDefault, + videoStreams: [ + { + index: 0, + height: 480, + width: 480, + codecName: 'h264', + frameCount: 100, + rotation: 0, + isHDR: false, + bitrate: 0, + pixelFormat: 'yuv420p10le', }, ], }), @@ -117,6 +139,7 @@ export const probeStub = { rotation: 90, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, ], }), @@ -132,6 +155,7 @@ export const probeStub = { rotation: 0, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, ], }), @@ -147,6 +171,7 @@ export const probeStub = { rotation: 0, isHDR: false, bitrate: 0, + pixelFormat: 'yuv420p', }, ], }), From 9dcd6e624dc02466eae52c27377721a77c81a760 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:22:18 -0400 Subject: [PATCH 2/7] remove npl setting --- docs/docs/install/config-file.md | 1 - i18n/en.json | 2 -- server/src/config.ts | 2 -- server/src/dtos/system-config.dto.ts | 6 ------ server/src/services/system-config.service.spec.ts | 1 - .../admin-page/settings/ffmpeg/ffmpeg-settings.svelte | 9 --------- 6 files changed, 21 deletions(-) diff --git a/docs/docs/install/config-file.md b/docs/docs/install/config-file.md index ed902f39cfd1e..24d747e93a629 100644 --- a/docs/docs/install/config-file.md +++ b/docs/docs/install/config-file.md @@ -26,7 +26,6 @@ The default configuration looks like this: "bframes": -1, "refs": 0, "gopSize": 0, - "npl": 0, "temporalAQ": false, "cqMode": "auto", "twoPass": false, diff --git a/i18n/en.json b/i18n/en.json index 7a1ada94a65c4..8de7d1d4e7af0 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -300,8 +300,6 @@ "transcoding_threads_description": "Higher values lead to faster encoding, but leave less room for the server to process other tasks while active. This value should not be more than the number of CPU cores. Maximizes utilization if set to 0.", "transcoding_tone_mapping": "Tone-mapping", "transcoding_tone_mapping_description": "Attempts to preserve the appearance of HDR videos when converted to SDR. Each algorithm makes different tradeoffs for color, detail and brightness. Hable preserves detail, Mobius preserves color, and Reinhard preserves brightness.", - "transcoding_tone_mapping_npl": "Tone-mapping NPL", - "transcoding_tone_mapping_npl_description": "Colors will be adjusted to look normal for a display of this brightness. Counter-intuitively, lower values increase the brightness of the video and vice versa since it compensates for the brightness of the display. 0 sets this value automatically.", "transcoding_transcode_policy": "Transcode policy", "transcoding_transcode_policy_description": "Policy for when a video should be transcoded. HDR videos will always be transcoded (except if transcoding is disabled).", "transcoding_two_pass_encoding": "Two-pass encoding", diff --git a/server/src/config.ts b/server/src/config.ts index fca6719bc0032..888b5adcfe087 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -35,7 +35,6 @@ export interface SystemConfig { bframes: number; refs: number; gopSize: number; - npl: number; temporalAQ: boolean; cqMode: CQMode; twoPass: boolean; @@ -170,7 +169,6 @@ export const defaults = Object.freeze({ bframes: -1, refs: 0, gopSize: 0, - npl: 0, temporalAQ: false, cqMode: CQMode.AUTO, twoPass: false, diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index 039dbd20ff36a..66a9561f5ec6a 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -110,12 +110,6 @@ export class SystemConfigFFmpegDto { @ApiProperty({ type: 'integer' }) gopSize!: number; - @IsInt() - @Min(0) - @Type(() => Number) - @ApiProperty({ type: 'integer' }) - npl!: number; - @ValidateBoolean() temporalAQ!: boolean; diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index f81abc4795d40..7638c18bf425e 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -58,7 +58,6 @@ const updatedConfig = Object.freeze({ bframes: -1, refs: 0, gopSize: 0, - npl: 0, temporalAQ: false, cqMode: CQMode.AUTO, twoPass: false, diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index 42cc004c52d21..8f5b587ae6bf2 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -343,15 +343,6 @@ subtitle={$t('admin.transcoding_advanced_options_description')} >
- - Date: Mon, 28 Oct 2024 16:24:20 -0400 Subject: [PATCH 3/7] update api --- .../openapi/lib/model/system_config_f_fmpeg_dto.dart | 11 +---------- open-api/immich-openapi-specs.json | 5 ----- open-api/typescript-sdk/src/fetch-client.ts | 1 - 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart index 73f7d35aecc30..0acfc9e8fbf9d 100644 --- a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart +++ b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart @@ -23,7 +23,6 @@ class SystemConfigFFmpegDto { required this.crf, required this.gopSize, required this.maxBitrate, - required this.npl, required this.preferredHwDevice, required this.preset, required this.refs, @@ -62,9 +61,6 @@ class SystemConfigFFmpegDto { String maxBitrate; - /// Minimum value: 0 - int npl; - String preferredHwDevice; String preset; @@ -102,7 +98,6 @@ class SystemConfigFFmpegDto { other.crf == crf && other.gopSize == gopSize && other.maxBitrate == maxBitrate && - other.npl == npl && other.preferredHwDevice == preferredHwDevice && other.preset == preset && other.refs == refs && @@ -128,7 +123,6 @@ class SystemConfigFFmpegDto { (crf.hashCode) + (gopSize.hashCode) + (maxBitrate.hashCode) + - (npl.hashCode) + (preferredHwDevice.hashCode) + (preset.hashCode) + (refs.hashCode) + @@ -142,7 +136,7 @@ class SystemConfigFFmpegDto { (twoPass.hashCode); @override - String toString() => 'SystemConfigFFmpegDto[accel=$accel, accelDecode=$accelDecode, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedContainers=$acceptedContainers, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, npl=$npl, preferredHwDevice=$preferredHwDevice, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]'; + String toString() => 'SystemConfigFFmpegDto[accel=$accel, accelDecode=$accelDecode, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedContainers=$acceptedContainers, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, preferredHwDevice=$preferredHwDevice, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]'; Map toJson() { final json = {}; @@ -156,7 +150,6 @@ class SystemConfigFFmpegDto { json[r'crf'] = this.crf; json[r'gopSize'] = this.gopSize; json[r'maxBitrate'] = this.maxBitrate; - json[r'npl'] = this.npl; json[r'preferredHwDevice'] = this.preferredHwDevice; json[r'preset'] = this.preset; json[r'refs'] = this.refs; @@ -190,7 +183,6 @@ class SystemConfigFFmpegDto { crf: mapValueOfType(json, r'crf')!, gopSize: mapValueOfType(json, r'gopSize')!, maxBitrate: mapValueOfType(json, r'maxBitrate')!, - npl: mapValueOfType(json, r'npl')!, preferredHwDevice: mapValueOfType(json, r'preferredHwDevice')!, preset: mapValueOfType(json, r'preset')!, refs: mapValueOfType(json, r'refs')!, @@ -259,7 +251,6 @@ class SystemConfigFFmpegDto { 'crf', 'gopSize', 'maxBitrate', - 'npl', 'preferredHwDevice', 'preset', 'refs', diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index a277156731329..acc7117249565 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -11581,10 +11581,6 @@ "maxBitrate": { "type": "string" }, - "npl": { - "minimum": 0, - "type": "integer" - }, "preferredHwDevice": { "type": "string" }, @@ -11633,7 +11629,6 @@ "crf", "gopSize", "maxBitrate", - "npl", "preferredHwDevice", "preset", "refs", diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 96ff24a8d785a..f0faf9583e3d4 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1095,7 +1095,6 @@ export type SystemConfigFFmpegDto = { crf: number; gopSize: number; maxBitrate: string; - npl: number; preferredHwDevice: string; preset: string; refs: number; From 70bee5c03e94e8becb50bec3c7a6a5169f66ac09 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Mon, 28 Oct 2024 19:17:15 -0400 Subject: [PATCH 4/7] handle variants of yuv420p --- server/src/utils/media.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index 06726943a79a1..f61b472b7512d 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -150,7 +150,7 @@ export class BaseConfig implements VideoCodecSWConfig { } options.push(...this.getToneMapping(videoStream)); - if (options.length === 0 && videoStream.pixelFormat !== 'yuv420p') { + if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) { options.push(`format=yuv420p`); } @@ -596,7 +596,7 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig { options.push(...this.getToneMapping(videoStream)); if (options.length > 0) { options[options.length - 1] += ':format=nv12'; - } else if (videoStream.pixelFormat !== 'yuv420p') { + } else if (!videoStream.pixelFormat.endsWith('420p')) { options.push('format=nv12'); } return options; @@ -743,7 +743,7 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig { options.push(scaling); } options.push(...tonemapOptions); - if (options.length === 0 && videoStream.pixelFormat !== 'yuv420p') { + if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) { options.push('format=nv12'); } return options; @@ -877,7 +877,7 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig { options.push(scaling); } options.push(...tonemapOptions); - if (options.length === 0 && videoStream.pixelFormat !== 'yuv420p') { + if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) { options.push('format=nv12'); } return options; From d2aa512c02174faf7e3f1207c310ec68a41a3ad6 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Mon, 28 Oct 2024 19:55:27 -0400 Subject: [PATCH 5/7] remove unused import --- server/src/services/media.service.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index b6ec01d9054de..706ad2515369b 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -9,7 +9,6 @@ import { AudioCodec, Colorspace, ImageFormat, - ToneMapping, TranscodeHWAccel, TranscodePolicy, VideoCodec, From 0f7094aff39774efd05cbf28d73a9a7571d4db64 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:44:42 -0400 Subject: [PATCH 6/7] migration to remove npl setting --- .../1730227312171-RemoveNplFromSystemConfig.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 server/src/migrations/1730227312171-RemoveNplFromSystemConfig.ts diff --git a/server/src/migrations/1730227312171-RemoveNplFromSystemConfig.ts b/server/src/migrations/1730227312171-RemoveNplFromSystemConfig.ts new file mode 100644 index 0000000000000..2c929191dd7e4 --- /dev/null +++ b/server/src/migrations/1730227312171-RemoveNplFromSystemConfig.ts @@ -0,0 +1,12 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class RemoveNplFromSystemConfig1730227312171 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + update system_metadata + set value = value #- '{ffmpeg,npl}' + where key = 'system-config' and value->'ffmpeg'->'npl' is not null`); + } + + public async down(): Promise {} +} From 05bbca23e979650cdc5847b9a07bad900eadbf6b Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:46:28 -0400 Subject: [PATCH 7/7] better pixel format check --- server/src/services/media.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index e6c7ebb1e124f..d7a1fa644315b 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -407,7 +407,7 @@ export class MediaService extends BaseService { const isLargerThanTargetBitrate = stream.bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate); const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(stream.codecName as VideoCodec); - const isRequired = !isTargetVideoCodec || stream.pixelFormat !== 'yuv420p'; + const isRequired = !isTargetVideoCodec || !stream.pixelFormat.endsWith('420p'); switch (ffmpegConfig.transcode) { case TranscodePolicy.DISABLED: {