Audio fixes

- For devices with >2 channels, only use first 2 when loading into
ffmpeg
- If output pipeline only has 1 audio source, don't use audio mixer

Author: Brendonovich

apps/web/app/s/[videoId]/_components/AuthOverlay.tsx

@@ -10,8 +10,8 @@ import { signIn } from "next-auth/react";
 import { useId, useState } from "react";
 import { toast } from "sonner";
 import { trackEvent } from "@/app/utils/analytics";
-import OtpForm from "./OtpForm";
 import { usePublicEnv } from "@/utils/public-env";
+import OtpForm from "./OtpForm";
 
 interface AuthOverlayProps {
 	isOpen: boolean;

apps/web/app/s/[videoId]/_components/Sidebar.tsx

@@ -97,7 +97,7 @@ export const Sidebar = forwardRef<{ scrollToBottom: () => void }, SidebarProps>(
 				: !(
 							videoSettings?.disableTranscript ??
 							data.orgSettings?.disableTranscript
-					  )
+						)
 					? "transcript"
 					: "activity";
 

crates/media-info/src/lib.rs

@@ -24,7 +24,7 @@ pub enum AudioInfoError {
 }
 
 impl AudioInfo {
-    pub const MAX_AUDIO_CHANNELS: u16 = 8;
+    pub const MAX_AUDIO_CHANNELS: u16 = 16;
 
     pub const fn new(
         sample_format: Sample,
@@ -133,18 +133,26 @@ impl AudioInfo {
         frame
     }
 
-    pub fn wrap_frame(&self, data: &[u8]) -> frame::Audio {
+    pub fn wrap_frame_with_max_channels(&self, data: &[u8], max_channels: usize) -> frame::Audio {
+        let out_channels = self.channels.min(max_channels);
+
         let sample_size = self.sample_size();
         let interleaved_chunk_size = sample_size * self.channels;
         let samples = data.len() / interleaved_chunk_size;
 
-        let mut frame = frame::Audio::new(self.sample_format, samples, self.channel_layout());
+        let mut frame = frame::Audio::new(
+            self.sample_format,
+            samples,
+            ChannelLayout::default(out_channels as i32),
+        );
         frame.set_rate(self.sample_rate);
 
         if self.channels == 0 {
             unreachable!()
-        } else if self.channels == 1 || frame.is_packed() {
+        } else if self.channels == 1 || (frame.is_packed() && self.channels <= max_channels) {
             frame.data_mut(0)[0..data.len()].copy_from_slice(data)
+        } else if frame.is_packed() && self.channels > max_channels {
+            todo!();
         } else {
             // cpal *always* returns interleaved data (i.e. the first sample from every channel, followed
             // by the second sample from every channel, et cetera). Many audio codecs work better/primarily
@@ -155,7 +163,7 @@ impl AudioInfo {
                 let start = chunk_index * sample_size;
                 let end = start + sample_size;
 
-                for channel in 0..self.channels {
+                for channel in 0..self.channels.min(max_channels) {
                     let channel_start = channel * sample_size;
                     let channel_end = channel_start + sample_size;
                     frame.data_mut(channel)[start..end]
@@ -166,6 +174,15 @@ impl AudioInfo {
 
         frame
     }
+
+    pub fn wrap_frame(&self, data: &[u8]) -> frame::Audio {
+        self.wrap_frame_with_max_channels(data, self.channels)
+    }
+
+    pub fn with_max_channels(mut self, channels: u16) -> Self {
+        self.channels = self.channels.min(channels as usize);
+        self
+    }
 }
 
 pub enum RawVideoFormat {

crates/recording/examples/recording-cli.rs

@@ -1,6 +1,7 @@
-use cap_recording::{screen_capture::ScreenCaptureTarget, *};
+use cap_recording::{feeds::*, screen_capture::ScreenCaptureTarget, *};
+use kameo::Actor as _;
 use scap_targets::Display;
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 use tracing::*;
 
 #[tokio::main]
@@ -37,23 +38,22 @@ pub async fn main() {
     //     .await
     //     .unwrap();
 
-    // let (error_tx, _) = flume::bounded(1);
-    // let mic_feed = MicrophoneFeed::spawn(MicrophoneFeed::new(error_tx));
-
-    // mic_feed
-    //     .ask(microphone::SetInput {
-    //         label:
-    //         // MicrophoneFeed::list()
-    //         //     .into_iter()
-    //         //     .find(|(k, _)| k.contains("Focusrite"))
-    //         MicrophoneFeed::default()
-    //             .map(|v| v.0)
-    //             .unwrap(),
-    //     })
-    //     .await
-    //     .unwrap()
-    //     .await
-    //     .unwrap();
+    let (error_tx, _) = flume::bounded(1);
+    let mic_feed = MicrophoneFeed::spawn(MicrophoneFeed::new(error_tx));
+
+    mic_feed
+        .ask(microphone::SetInput {
+            label: MicrophoneFeed::list()
+                .into_iter()
+                .find(|(k, _)| k.contains("BlackHole"))
+                // MicrophoneFeed::default_device()
+                .map(|v| v.0)
+                .unwrap(),
+        })
+        .await
+        .unwrap()
+        .await
+        .unwrap();
 
     tokio::time::sleep(Duration::from_millis(10)).await;
 
@@ -63,10 +63,11 @@ pub async fn main() {
             id: Display::primary().id(),
         },
     )
-    // .with_system_audio(true)
+    .with_system_audio(true)
     // .with_camera_feed(std::sync::Arc::new(
     //     camera_feed.ask(feeds::camera::Lock).await.unwrap(),
     // ))
+    .with_mic_feed(Arc::new(mic_feed.ask(microphone::Lock).await.unwrap()))
     .build(
         #[cfg(target_os = "macos")]
         cidre::sc::ShareableContent::current().await.unwrap(),