fix: determine isAnamorphic via DAR/SAR; better square pixel size cal…

…culations
chrisbenincasa · Jan 6, 2025 · d3dd7e4 · d3dd7e4
1 parent dea698c
commit d3dd7e4
Show file tree

Hide file tree

Showing 9 changed files with 96 additions and 28 deletions.
diff --git a/server/src/ffmpeg/FfmpegStreamFactory.ts b/server/src/ffmpeg/FfmpegStreamFactory.ts
@@ -165,9 +165,9 @@ export class FfmpegStreamFactory extends IFFMPEG {
       codec: VideoFormats.Raw,
       pixelFormat: new PixelFormatYuv420P(), // Hard-coded right now
       frameSize: FrameSize.fromResolution(this.ffmpegSettings.targetResolution),
-      pixelAspectRatio: '1:1',
+      sampleAspectRatio: '1:1',
+      displayAspectRatio: '1:1',
       inputKind: 'video',
-      isAnamorphic: false,
     });
 
     const videoInputSource = VideoInputSource.withStream(
@@ -314,9 +314,9 @@ export class FfmpegStreamFactory extends IFFMPEG {
       codec: videoStreamDetails.codec ?? 'unknown',
       profile: videoStreamDetails.profile,
       index: isNaN(streamIndex) ? 0 : streamIndex,
-      isAnamorphic: videoStreamDetails.anamorphic ?? false,
       inputKind: 'video',
-      pixelAspectRatio: null,
+      sampleAspectRatio: videoStreamDetails.sampleAspectRatio ?? null,
+      displayAspectRatio: videoStreamDetails.displayAspectRatio,
       pixelFormat,
       frameSize: FrameSize.create({
         height: videoStreamDetails.height,
@@ -474,8 +474,8 @@ export class FfmpegStreamFactory extends IFFMPEG {
             codec: 'unknown',
             frameSize: FrameSize.create({ width: 1920, height: 1080 }),
             index: 0,
-            isAnamorphic: false,
-            pixelAspectRatio: null,
+            sampleAspectRatio: '1:1',
+            displayAspectRatio: '1:1',
             pixelFormat: PixelFormatUnknown(),
           }),
         );
@@ -592,8 +592,8 @@ export class FfmpegStreamFactory extends IFFMPEG {
         codec: 'unknown',
         frameSize: FrameSize.create({ width: 1920, height: 1080 }),
         index: 0,
-        isAnamorphic: false,
-        pixelAspectRatio: null,
+        sampleAspectRatio: '1:1',
+        displayAspectRatio: '1:1',
         pixelFormat: PixelFormatUnknown(),
       }),
     );

diff --git a/server/src/ffmpeg/builder/FfmpegCommandGenerator.test.ts b/server/src/ffmpeg/builder/FfmpegCommandGenerator.test.ts
@@ -59,7 +59,7 @@ describe('FfmpegCommandGenerator', () => {
       pixelFormat,
       frameSize: FrameSize.create({ width: 640, height: 480 }),
       isAnamorphic: false,
-      pixelAspectRatio: null,
+      sampleAspectRatio: null,
     });
 
     const audioState = AudioState.create({

diff --git a/server/src/ffmpeg/builder/MediaStream.test.ts b/server/src/ffmpeg/builder/MediaStream.test.ts
@@ -0,0 +1,18 @@
+import { VideoStream } from '@/ffmpeg/builder/MediaStream.ts';
+import { PixelFormatYuv420P } from '@/ffmpeg/builder/format/PixelFormat.ts';
+import { FrameSize } from '@/ffmpeg/builder/types.ts';
+
+describe('MediaStream', () => {
+  test('squarePixelFrameSize @ FHD', () => {
+    const stream = VideoStream.create({
+      codec: 'h264',
+      frameSize: FrameSize.withDimensions(720, 480),
+      index: 0,
+      sampleAspectRatio: null,
+      displayAspectRatio: '1.33',
+      pixelFormat: new PixelFormatYuv420P(),
+    });
+
+    console.log(stream.squarePixelFrameSize(FrameSize.FHD));
+  });
+});
diff --git a/server/src/ffmpeg/builder/MediaStream.ts b/server/src/ffmpeg/builder/MediaStream.ts
@@ -1,4 +1,5 @@
 import { ExcludeByValueType, Nullable } from '@/types/util.ts';
+import { isNonEmptyString } from '@/util/index.ts';
 import { isNull, merge } from 'lodash-es';
 import { AnyFunction, MarkOptional } from 'ts-essentials';
 import { PixelFormat } from './format/PixelFormat.ts';
@@ -22,7 +23,7 @@ type MediaStreamFields<T extends MediaStream> = Omit<
 // semantics with class construction, but still enabling us
 // to have hierarchies, methods, etc.
 type AudioStreamFields = MediaStreamFields<AudioStream>;
-type VideoStreamFields = MediaStreamFields<VideoStream>;
+type VideoStreamFields = Omit<MediaStreamFields<VideoStream>, 'isAnamorphic'>;
 
 export class AudioStream implements MediaStream {
   readonly kind: StreamKind = 'audio';
@@ -51,9 +52,9 @@ export class VideoStream implements MediaStream {
   pixelFormat: Nullable<PixelFormat>;
   frameSize: FrameSize;
   frameRate?: string;
-  isAnamorphic: boolean;
-  pixelAspectRatio: Nullable<`${number}:${number}`>;
   inputKind: VideoInputKind = 'video' as const;
+  sampleAspectRatio: Nullable<string>;
+  displayAspectRatio: string;
 
   protected constructor(fields: MarkOptional<VideoStreamFields, 'inputKind'>) {
     // Unfortunately TS is not 'smart' enough to let us
@@ -71,17 +72,30 @@ export class VideoStream implements MediaStream {
     return this.pixelFormat?.bitDepth ?? 8;
   }
 
+  get isAnamorphic() {
+    if (this.sampleAspectRatio === '1:1') {
+      return false;
+    } else if (this.sampleAspectRatio !== '0:1') {
+      return true;
+    } else if (this.displayAspectRatio === '0:1') {
+      return false;
+    }
+
+    return (
+      this.displayAspectRatio !==
+      `${this.frameSize.width}:${this.frameSize.height}`
+    );
+  }
+
   squarePixelFrameSize(resolution: FrameSize): FrameSize {
     let width = this.frameSize.width;
     let height = this.frameSize.height;
 
-    if (this.isAnamorphic && !isNull(this.pixelAspectRatio)) {
-      const [numStr, denStr] = this.pixelAspectRatio.split(':');
-      const num = Number.parseFloat(numStr);
-      const den = Number.parseFloat(denStr);
+    if (this.isAnamorphic && !isNull(this.sampleAspectRatio)) {
+      const sar = this.getSampleAspectRatio();
 
-      width = Math.floor((this.frameSize.width * num) / den);
-      height = Math.floor((this.frameSize.height * num) / den);
+      width = Math.floor(this.frameSize.width * sar);
+      height = Math.floor(this.frameSize.height * sar);
     }
 
     const widthPercent = resolution.width / width;
@@ -93,6 +107,25 @@ export class VideoStream implements MediaStream {
       height: Math.floor(height * minPercent),
     });
   }
+
+  private getSampleAspectRatio() {
+    if (
+      !isNonEmptyString(this.sampleAspectRatio) ||
+      this.sampleAspectRatio === '0:0'
+    ) {
+      let dar = parseFloat(this.displayAspectRatio);
+      if (isNaN(dar)) {
+        const [num, den] = this.displayAspectRatio.split(':');
+        dar = parseFloat(num) / parseFloat(den);
+      }
+
+      const res = this.frameSize.width / this.frameSize.height;
+      return dar / res;
+    }
+
+    const [num, den] = this.sampleAspectRatio.split(':');
+    return parseFloat(num) / parseFloat(den);
+  }
 }
 
 type StillImageStreamFields = Omit<
@@ -101,7 +134,8 @@ type StillImageStreamFields = Omit<
   | 'kind'
   | 'pixelFormat'
   | 'isAnamorphic'
-  | 'pixelAspectRatio'
+  | 'sampleAspectRatio'
+  | 'displayAspectRatio'
   | 'inputKind'
 >;
 
@@ -112,11 +146,10 @@ export class StillImageStream extends VideoStream {
     super({
       ...fields,
       codec: '',
-      isAnamorphic: false,
-      pixelAspectRatio: null,
+      sampleAspectRatio: '1:1',
+      displayAspectRatio: '1:1',
       pixelFormat: null,
     });
-    // merge(this, fields);
   }
 
   static create(fields: StillImageStreamFields) {

diff --git a/server/src/ffmpeg/builder/filter/qsv/ScaleQsvFilter.ts b/server/src/ffmpeg/builder/filter/qsv/ScaleQsvFilter.ts
@@ -32,7 +32,7 @@ export class ScaleQsvFilter extends FilterOption {
     if (!this.currentState.scaledSize.equals(this.scaledSize)) {
       const targetSize = `w=${this.scaledSize.width}:h=${this.scaledSize.height}`;
       const sarValue =
-        this.videoStream.pixelAspectRatio?.replace(':', '/') ?? '1/1';
+        this.videoStream.sampleAspectRatio?.replace(':', '/') ?? '1/1';
       let squareScale = '';
       let format = '';
       if (this.currentState.isAnamorphic) {

diff --git a/server/src/ffmpeg/builder/input/ConcatInputSource.ts b/server/src/ffmpeg/builder/input/ConcatInputSource.ts
@@ -26,8 +26,8 @@ export class ConcatInputSource extends InputSource<VideoStream> {
         profile: '',
         pixelFormat: null,
         frameSize: this.frameSize,
-        isAnamorphic: false,
-        pixelAspectRatio: null,
+        sampleAspectRatio: null,
+        displayAspectRatio: '1:1',
         inputKind: 'video',
       }),
     ];

diff --git a/server/src/ffmpeg/builder/input/LavfiVideoInputSource.ts b/server/src/ffmpeg/builder/input/LavfiVideoInputSource.ts
@@ -18,10 +18,10 @@ export class LavfiVideoInputSource extends VideoInputSource {
       VideoStream.create({
         codec: 'generated',
         pixelFormat: PixelFormatUnknown(),
-        isAnamorphic: false,
         index: 0,
         inputKind: 'filter',
-        pixelAspectRatio: null,
+        sampleAspectRatio: null,
+        displayAspectRatio: '1:1',
         frameSize: size,
       }),
     ]);

diff --git a/server/src/ffmpeg/builder/pipeline/software/SoftwarePipelineBuilder.test.ts b/server/src/ffmpeg/builder/pipeline/software/SoftwarePipelineBuilder.test.ts
@@ -20,7 +20,8 @@ describe('SoftwarePipelineBuilder', () => {
       pixelFormat,
       frameSize: FrameSize.create({ width: 640, height: 480 }),
       isAnamorphic: false,
-      pixelAspectRatio: null,
+      sampleAspectRatio: null,
+      displayAspectRatio: '1:1',
     });
 
     const audioState = AudioState.create({

diff --git a/server/src/util/strings.ts b/server/src/util/strings.ts
@@ -1,3 +1,6 @@
+import { Maybe } from '@/types/util.ts';
+import { isEmpty } from 'lodash-es';
+
 /**
  * Performs a naive sanitization for passing user-inputted text to
  * child_process exec:
@@ -15,3 +18,16 @@ export function sanitizeForExec(executable: string): string {
 
   return cleaned.trim().replace(/\s+/g, ' ');
 }
+
+export function trimToUndefined(s: Maybe<string>): Maybe<string> {
+  if (!s) {
+    return;
+  }
+
+  const trim = s.trim();
+  if (isEmpty(trim)) {
+    return;
+  }
+
+  return s;
+}