account for screen capture not providing frames on macos

Author: Brendonovich

crates/enc-avfoundation/src/mp4.rs

@@ -2,7 +2,7 @@ use cap_media_info::{AudioInfo, VideoInfo};
 use cidre::{cm::SampleTimingInfo, objc::Obj, *};
 use ffmpeg::frame;
 use std::{ops::Sub, path::PathBuf, time::Duration};
-use tracing::{debug, info};
+use tracing::{debug, error, info};
 
 // before pausing at all, subtract 0.
 // on pause, record last frame time.
@@ -16,7 +16,7 @@ pub struct MP4Encoder {
     asset_writer: arc::R<av::AssetWriter>,
     video_input: arc::R<av::AssetWriterInput>,
     audio_input: Option<arc::R<av::AssetWriterInput>>,
-    most_recent_timestamp: Option<Duration>,
+    most_recent_frame: Option<(arc::R<cm::SampleBuf>, Duration)>,
     pause_timestamp: Option<Duration>,
     timestamp_offset: Duration,
     is_writing: bool,
@@ -197,7 +197,7 @@ impl MP4Encoder {
             audio_input,
             asset_writer,
             video_input,
-            most_recent_timestamp: None,
+            most_recent_frame: None,
             pause_timestamp: None,
             timestamp_offset: Duration::ZERO,
             is_writing: false,
@@ -211,7 +211,7 @@ impl MP4Encoder {
     /// They will be made relative when encoding
     pub fn queue_video_frame(
         &mut self,
-        frame: &cidre::cm::SampleBuf,
+        frame: arc::R<cm::SampleBuf>,
         timestamp: Duration,
     ) -> Result<(), QueueVideoFrameError> {
         if self.is_paused || !self.video_input.is_ready_for_more_media_data() {
@@ -224,7 +224,7 @@ impl MP4Encoder {
                 .start_session_at_src_time(cm::Time::new(timestamp.as_millis() as i64, 1_000));
         }
 
-        self.most_recent_timestamp = Some(timestamp);
+        self.most_recent_frame = Some((frame.clone(), timestamp));
 
         if let Some(pause_timestamp) = self.pause_timestamp {
             self.timestamp_offset += timestamp - pause_timestamp;
@@ -331,7 +331,7 @@ impl MP4Encoder {
             return;
         }
 
-        let Some(timestamp) = self.most_recent_timestamp else {
+        let Some((_, timestamp)) = self.most_recent_frame else {
             return;
         };
 
@@ -347,19 +347,34 @@ impl MP4Encoder {
         self.is_paused = false;
     }
 
-    pub fn finish(&mut self) {
+    pub fn finish(&mut self, timestamp: Option<Duration>) {
         if !self.is_writing {
             return;
         }
 
-        let Some(most_recent_timestamp) = self.most_recent_timestamp else {
+        let Some(mut most_recent_frame) = self.most_recent_frame.take() else {
             return;
         };
 
+        // We extend the video to the provided timestamp if possible
+        if let Some(timestamp) = timestamp
+            && let Some(diff) = timestamp.checked_sub(most_recent_frame.1)
+            && diff > Duration::from_millis(500)
+        {
+            match self.queue_video_frame(most_recent_frame.0.clone(), timestamp) {
+                Ok(()) => {
+                    most_recent_frame = (most_recent_frame.0, timestamp);
+                }
+                Err(e) => {
+                    error!("Failed to queue final video frame: {e}");
+                }
+            }
+        }
+
         self.is_writing = false;
 
         self.asset_writer.end_session_at_src_time(cm::Time::new(
-            most_recent_timestamp.sub(self.timestamp_offset).as_millis() as i64,
+            most_recent_frame.1.sub(self.timestamp_offset).as_millis() as i64,
             1000,
         ));
         self.video_input.mark_as_finished();
@@ -381,7 +396,7 @@ impl MP4Encoder {
 
 impl Drop for MP4Encoder {
     fn drop(&mut self) {
-        self.finish();
+        self.finish(None);
     }
 }
 

crates/recording/src/output_pipeline/core.rs

@@ -349,7 +349,7 @@ async fn finish_build(
             Ok(())
         }
         .then(async move |res| {
-            let muxer_res = muxer.lock().await.finish();
+            let muxer_res = muxer.lock().await.finish(timestamps.instant().elapsed());
 
             let _ = done_tx.send(match (res, muxer_res) {
                 (Err(e), _) | (_, Err(e)) => Err(e),
@@ -766,7 +766,7 @@ pub trait Muxer: Send + 'static {
 
     fn stop(&mut self) {}
 
-    fn finish(&mut self) -> anyhow::Result<()>;
+    fn finish(&mut self, timestamp: Duration) -> anyhow::Result<()>;
 }
 
 pub trait AudioMuxer: Muxer {

crates/recording/src/output_pipeline/ffmpeg.rs

@@ -65,7 +65,7 @@ impl Muxer for Mp4Muxer {
         })
     }
 
-    fn finish(&mut self) -> anyhow::Result<()> {
+    fn finish(&mut self, _: Duration) -> anyhow::Result<()> {
         if let Some(video_encoder) = self.video_encoder.as_mut() {
             video_encoder.finish(&mut self.output);
         }
@@ -131,7 +131,7 @@ impl Muxer for OggMuxer {
         ))
     }
 
-    fn finish(&mut self) -> anyhow::Result<()> {
+    fn finish(&mut self, _: Duration) -> anyhow::Result<()> {
         self.0.finish();
         Ok(())
     }

crates/recording/src/output_pipeline/macos.rs

@@ -49,8 +49,11 @@ impl Muxer for AVFoundationMp4Muxer {
         ))
     }
 
-    fn finish(&mut self) -> anyhow::Result<()> {
-        self.0.lock().map_err(|e| anyhow!("{e}"))?.finish();
+    fn finish(&mut self, timestamp: Duration) -> anyhow::Result<()> {
+        self.0
+            .lock()
+            .map_err(|e| anyhow!("{e}"))?
+            .finish(Some(timestamp));
         Ok(())
     }
 }
@@ -71,7 +74,7 @@ impl VideoMuxer for AVFoundationMp4Muxer {
             mp4.resume();
         }
 
-        mp4.queue_video_frame(&frame.sample_buf, timestamp)
+        mp4.queue_video_frame(frame.sample_buf, timestamp)
             .map_err(|e| anyhow!("QueueVideoFrame/{e}"))
     }
 }