From db77e0775964487bbe7d63b524dd7feb6cf99a7b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:38:01 +0900 Subject: [PATCH 1/4] Debug bar --- .../contents.xcworkspacedata | 13 + VoiceAssistant/AudioSessionObserverView.swift | 302 ++++++++++++++++++ VoiceAssistant/ContentView.swift | 98 ++++++ 3 files changed, 413 insertions(+) create mode 100644 VoiceAssistant-dev.xcworkspace/contents.xcworkspacedata create mode 100644 VoiceAssistant/AudioSessionObserverView.swift diff --git a/VoiceAssistant-dev.xcworkspace/contents.xcworkspacedata b/VoiceAssistant-dev.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..0679f85 --- /dev/null +++ b/VoiceAssistant-dev.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,13 @@ + + + + + + + + + diff --git a/VoiceAssistant/AudioSessionObserverView.swift b/VoiceAssistant/AudioSessionObserverView.swift new file mode 100644 index 0000000..9ecb770 --- /dev/null +++ b/VoiceAssistant/AudioSessionObserverView.swift @@ -0,0 +1,302 @@ +// +// AudioSessionObserverView.swift +// VoiceAssistant +// +// Created by Hiroshi Horie on 2025/03/17. +// + +import AVFoundation +import SwiftUI + +class AudioSessionObserver: NSObject, ObservableObject { + @Published var currentCategory: AVAudioSession.Category = .playback + @Published var currentMode: AVAudioSession.Mode = .default + @Published var currentCategoryOptions: AVAudioSession.CategoryOptions = [] + @Published var isActive: Bool = false + @Published var inputLatency: TimeInterval = 0 + @Published var outputLatency: TimeInterval = 0 + @Published var sampleRate: Double = 0 + @Published var preferredSampleRate: Double = 0 + @Published var inputNumberOfChannels: Int = 0 + @Published var outputNumberOfChannels: Int = 0 + @Published var maximumInputNumberOfChannels: Int = 0 + @Published var maximumOutputNumberOfChannels: Int = 0 + @Published var preferredInputNumberOfChannels: Int = 0 + @Published var preferredOutputNumberOfChannels: Int = 0 + @Published var interruptionCount: Int = 0 + @Published var routeChangeCount: Int = 0 + @Published var lastInterruptionType: String = "None" + @Published var lastRouteChangeReason: String = "None" + + private var observationTokens: [NSObjectProtocol] = [] + + override init() { + super.init() + setupNotifications() + updateSessionInfo() + } + + deinit { + // Remove all notification observers + observationTokens.forEach { NotificationCenter.default.removeObserver($0) } + } + + private func setupNotifications() { + let center = NotificationCenter.default + + let interruptionToken = center.addObserver( + forName: AVAudioSession.interruptionNotification, + object: nil, + queue: .main + ) { [weak self] notification in + self?.handleInterruption(notification) + } + + let routeChangeToken = center.addObserver( + forName: AVAudioSession.routeChangeNotification, + object: nil, + queue: .main + ) { [weak self] notification in + self?.handleRouteChange(notification) + } + + let mediaServicesResetToken = center.addObserver( + forName: AVAudioSession.mediaServicesWereResetNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.handleMediaServicesReset() + } + + let silenceSecondaryAudioToken = center.addObserver( + forName: AVAudioSession.silenceSecondaryAudioHintNotification, + object: nil, + queue: .main + ) { [weak self] notification in + self?.handleSilenceSecondaryAudio(notification) + } + + observationTokens = [interruptionToken, routeChangeToken, mediaServicesResetToken, silenceSecondaryAudioToken] + } + + func updateSessionInfo() { + let session = AVAudioSession.sharedInstance() + currentCategory = session.category + currentMode = session.mode + currentCategoryOptions = session.categoryOptions + inputLatency = session.inputLatency + outputLatency = session.outputLatency + sampleRate = session.sampleRate + preferredSampleRate = session.preferredSampleRate + inputNumberOfChannels = session.inputNumberOfChannels + outputNumberOfChannels = session.outputNumberOfChannels + maximumInputNumberOfChannels = session.maximumInputNumberOfChannels + maximumOutputNumberOfChannels = session.maximumOutputNumberOfChannels + preferredInputNumberOfChannels = session.preferredInputNumberOfChannels + preferredOutputNumberOfChannels = session.preferredOutputNumberOfChannels + } + + // Handle interruption notifications + private func handleInterruption(_ notification: Notification) { + guard let userInfo = notification.userInfo, + let typeValue = userInfo[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = AVAudioSession.InterruptionType(rawValue: typeValue) + else { + return + } + + interruptionCount += 1 + + switch type { + case .began: + lastInterruptionType = "Notification: Began" + print("Audio Session interruption began") + case .ended: + if let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt { + let options = AVAudioSession.InterruptionOptions(rawValue: optionsValue) + if options.contains(.shouldResume) { + lastInterruptionType = "Notification: Ended (Should Resume)" + print("Audio Session interruption ended with options: Should Resume") + } else { + lastInterruptionType = "Notification: Ended" + print("Audio Session interruption ended") + } + } else { + lastInterruptionType = "Notification: Ended" + print("Audio Session interruption ended") + } + @unknown default: + lastInterruptionType = "Notification: Unknown" + } + + updateSessionInfo() + } + + // Handle route change notifications + private func handleRouteChange(_ notification: Notification) { + guard let userInfo = notification.userInfo, + let reasonValue = userInfo[AVAudioSessionRouteChangeReasonKey] as? UInt, + let reason = AVAudioSession.RouteChangeReason(rawValue: reasonValue) + else { + return + } + + routeChangeCount += 1 + + switch reason { + case .newDeviceAvailable: + lastRouteChangeReason = "New Device Available" + case .oldDeviceUnavailable: + lastRouteChangeReason = "Old Device Unavailable" + case .categoryChange: + lastRouteChangeReason = "Category Change" + case .override: + lastRouteChangeReason = "Override" + case .wakeFromSleep: + lastRouteChangeReason = "Wake From Sleep" + case .noSuitableRouteForCategory: + lastRouteChangeReason = "No Suitable Route For Category" + case .routeConfigurationChange: + lastRouteChangeReason = "Route Configuration Change" + case .unknown: + lastRouteChangeReason = "Unknown" + @unknown default: + lastRouteChangeReason = "Unknown (\(reasonValue))" + } + + updateSessionInfo() + } + + // Handle media services reset + private func handleMediaServicesReset() { + print("Media services were reset") + updateSessionInfo() + } + + // Handle silence secondary audio hint + private func handleSilenceSecondaryAudio(_ notification: Notification) { + guard let userInfo = notification.userInfo, + let typeValue = userInfo[AVAudioSessionSilenceSecondaryAudioHintTypeKey] as? UInt, + let type = AVAudioSession.SilenceSecondaryAudioHintType(rawValue: typeValue) + else { + return + } + + switch type { + case .begin: + print("Secondary audio did begin") + case .end: + print("Secondary audio did end") + @unknown default: + print("Unknown secondary audio hint type") + } + + updateSessionInfo() + } + + // Method to change audio session category and mode + func setSessionCategory(_ category: AVAudioSession.Category, mode: AVAudioSession.Mode, options: AVAudioSession.CategoryOptions = []) { + do { + let session = AVAudioSession.sharedInstance() + try session.setCategory(category, mode: mode, options: options) + try session.setActive(true) + updateSessionInfo() + } catch { + print("Failed to set audio session category: \(error)") + } + } +} + +struct AudioSessionMonitorView: View { + @StateObject private var sessionObserver = AudioSessionObserver() + @State private var showDetails = false + + private func categoryName(for category: AVAudioSession.Category) -> String { + switch category { + case .ambient: return "Ambient" + case .soloAmbient: return "Solo Ambient" + case .playback: return "Playback" + case .record: return "Record" + case .playAndRecord: return "Play and Record" + case .multiRoute: return "Multi-Route" + default: return "Unknown" + } + } + + private func modeName(for mode: AVAudioSession.Mode) -> String { + switch mode { + case .default: return "Default" + case .gameChat: return "Game Chat" + case .measurement: return "Measurement" + case .moviePlayback: return "Movie Playback" + case .spokenAudio: return "Spoken Audio" + case .videoChat: return "Video Chat" + case .videoRecording: return "Video Recording" + case .voiceChat: return "Voice Chat" + case .voicePrompt: return "Voice Prompt" + default: return "Unknown" + } + } + + private func categoryOptionsDescription(_ options: AVAudioSession.CategoryOptions) -> String { + var descriptions: [String] = [] + + if options.contains(.mixWithOthers) { + descriptions.append("Mix With Others") + } + if options.contains(.duckOthers) { + descriptions.append("Duck Others") + } + if options.contains(.allowBluetooth) { + descriptions.append("Allow Bluetooth") + } + if options.contains(.defaultToSpeaker) { + descriptions.append("Default To Speaker") + } + if options.contains(.interruptSpokenAudioAndMixWithOthers) { + descriptions.append("Interrupt Spoken Audio") + } + if options.contains(.allowBluetoothA2DP) { + descriptions.append("Allow Bluetooth A2DP") + } + if options.contains(.allowAirPlay) { + descriptions.append("Allow AirPlay") + } + + return descriptions.isEmpty ? "None" : descriptions.joined(separator: ", ") + } + + var body: some View { + List { + // Current Session Information + Section(header: Text("Current Audio Session")) { + HStack { + Text("Category") + Spacer() + Text(categoryName(for: sessionObserver.currentCategory)) + .foregroundColor(.secondary) + } + + HStack { + Text("Mode") + Spacer() + Text(modeName(for: sessionObserver.currentMode)) + .foregroundColor(.secondary) + } + + HStack { + Text("Options") + Spacer() + Text(categoryOptionsDescription(sessionObserver.currentCategoryOptions)) + .font(.caption) + .foregroundColor(.secondary) + .multilineTextAlignment(.trailing) + } + } + } + .navigationTitle("Audio Session Monitor") + .onAppear { + sessionObserver.updateSessionInfo() + } + } +} diff --git a/VoiceAssistant/ContentView.swift b/VoiceAssistant/ContentView.swift index 9764c44..722753b 100644 --- a/VoiceAssistant/ContentView.swift +++ b/VoiceAssistant/ContentView.swift @@ -1,3 +1,4 @@ +import AVFAudio import LiveKit import SwiftUI #if os(iOS) || os(macOS) @@ -26,6 +27,7 @@ struct ContentView: View { .frame(maxWidth: 512) ControlBar() + DebugBar() } .padding() .environmentObject(room) @@ -36,3 +38,99 @@ struct ContentView: View { } } } + +struct DebugBar: View { + enum MusicPlayerState { + case stopped + case downloading + case playing + } + + @State var musicPlayerState: MusicPlayerState = .stopped + + @State var isVoiceProcessingEnabled: Bool = true + @State var isVoiceProcessingBypassed: Bool = false + @State var isRecordingAlwaysPrepared: Bool = false + + @State private var player: AVAudioPlayer? + @State private var localMusicFile: URL? + + var body: some View { + AudioSessionMonitorView() + + Toggle(isOn: $isVoiceProcessingEnabled) { + Text("Voice processing enabled") + }.onChange(of: isVoiceProcessingEnabled) { _, _ in + print("Setting voice processing enabled: \(isVoiceProcessingEnabled)") + Task { + do { + try AudioManager.shared.setVoiceProcessingEnabled(isVoiceProcessingEnabled) + } catch { + print("Failed to set voice processing enabled: \(error)") + } + } + } + + Toggle(isOn: $isVoiceProcessingBypassed) { + Text("Voice processing bypassed") + }.onChange(of: isVoiceProcessingBypassed) { _, _ in + print("Setting voice processing bypassed: \(isVoiceProcessingBypassed)") + AudioManager.shared.isVoiceProcessingBypassed = isVoiceProcessingBypassed + } + + // Prepare recording switch + Toggle(isOn: $isRecordingAlwaysPrepared) { + Text("Recording prepared") + }.onChange(of: isRecordingAlwaysPrepared) { _, _ in + print("Setting recording always prepared mode: \(isRecordingAlwaysPrepared)") + Task { + do { + try AudioManager.shared.setRecordingAlwaysPreparedMode(isRecordingAlwaysPrepared) + } catch { + print("Failed to set recording always prepared mode: \(error)") + } + } + } + + // BG Music player + if musicPlayerState == .stopped { + Button { + Task { + do { + if localMusicFile == nil { + musicPlayerState = .downloading + print("Downloading music file...") + let url = URL(string: "https://upload.wikimedia.org/wikipedia/commons/b/b0/Free_Man_-_Wild_Blue_Country_-_United_States_Air_Force_Academy_Band.mp3")! + let (localUrl, _) = try await URLSession.shared.download(from: url) + localMusicFile = localUrl + } + guard let localMusicFile else { return } + print("Playing music file...") + let player = try AVAudioPlayer(contentsOf: localMusicFile) + player.volume = 1.0 + player.play() + musicPlayerState = .playing + self.player = player + } catch { + print("Failed to play music file: \(error)") + } + } + } label: { + Text("Play Music") + } + } else if musicPlayerState == .downloading { + HStack { + ProgressView() + Text("Downloading...") + } + } else if musicPlayerState == .playing { + Button { + player?.stop() + player = nil + musicPlayerState = .stopped + } label: { + Text("Stop Music") + } + } + } +} From 542fe34117c07520ba5119cd0bd56e7a62997667 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 19 Mar 2025 07:49:48 +0800 Subject: [PATCH 2/4] ducking picker --- VoiceAssistant.xcodeproj/project.pbxproj | 2 + VoiceAssistant/ContentView.swift | 48 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/VoiceAssistant.xcodeproj/project.pbxproj b/VoiceAssistant.xcodeproj/project.pbxproj index 4749ecb..715b568 100644 --- a/VoiceAssistant.xcodeproj/project.pbxproj +++ b/VoiceAssistant.xcodeproj/project.pbxproj @@ -312,6 +312,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; + SUPPORTS_MACCATALYST = YES; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2,7"; @@ -356,6 +357,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; + SUPPORTS_MACCATALYST = YES; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2,7"; diff --git a/VoiceAssistant/ContentView.swift b/VoiceAssistant/ContentView.swift index 722753b..d5355d1 100644 --- a/VoiceAssistant/ContentView.swift +++ b/VoiceAssistant/ContentView.swift @@ -39,6 +39,29 @@ struct ContentView: View { } } +extension AudioDuckingLevel: @retroactive Identifiable { + public var id: Int { + rawValue + } +} + +extension AudioDuckingLevel: @retroactive CustomStringConvertible { + public var description: String { + switch self { + case .default: + return "Default" + case .min: + return "Min" + case .mid: + return "Mid" + case .max: + return "Max" + @unknown default: + return "Unknown" + } + } +} + struct DebugBar: View { enum MusicPlayerState { case stopped @@ -52,6 +75,14 @@ struct DebugBar: View { @State var isVoiceProcessingBypassed: Bool = false @State var isRecordingAlwaysPrepared: Bool = false + let duckingLevels: [AudioDuckingLevel] = [ + .default, + .min, + .mid, + .max, + ] + @State var otherAudioDuckingLevel: AudioDuckingLevel = .default + @State private var player: AVAudioPlayer? @State private var localMusicFile: URL? @@ -78,6 +109,17 @@ struct DebugBar: View { AudioManager.shared.isVoiceProcessingBypassed = isVoiceProcessingBypassed } + Picker("Audio Ducking Level", selection: $otherAudioDuckingLevel) { + ForEach(duckingLevels, id: \.self) { option in + Text(String(describing: option)) + .tag(option) + } + } + .onChange(of: otherAudioDuckingLevel) { _, newValue in + print("Setting other audio ducking level: \(newValue)") + AudioManager.shared.duckingLevel = newValue + } + // Prepare recording switch Toggle(isOn: $isRecordingAlwaysPrepared) { Text("Recording prepared") @@ -109,6 +151,12 @@ struct DebugBar: View { let player = try AVAudioPlayer(contentsOf: localMusicFile) player.volume = 1.0 player.play() + + // Low volume workaround + // https://developer.apple.com/forums/thread/721535 + try AVAudioSession.sharedInstance().overrideOutputAudioPort(.speaker) + try AVAudioSession.sharedInstance().overrideOutputAudioPort(.none) + musicPlayerState = .playing self.player = player } catch { From a1e703a4ca69b1f94b48964910bff0c34005e669 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 19 Mar 2025 09:55:30 +0800 Subject: [PATCH 3/4] Only iOS --- VoiceAssistant/AudioSessionObserverView.swift | 2 ++ VoiceAssistant/ContentView.swift | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/VoiceAssistant/AudioSessionObserverView.swift b/VoiceAssistant/AudioSessionObserverView.swift index 9ecb770..7640123 100644 --- a/VoiceAssistant/AudioSessionObserverView.swift +++ b/VoiceAssistant/AudioSessionObserverView.swift @@ -8,6 +8,7 @@ import AVFoundation import SwiftUI +#if os(iOS) class AudioSessionObserver: NSObject, ObservableObject { @Published var currentCategory: AVAudioSession.Category = .playback @Published var currentMode: AVAudioSession.Mode = .default @@ -300,3 +301,4 @@ struct AudioSessionMonitorView: View { } } } +#endif diff --git a/VoiceAssistant/ContentView.swift b/VoiceAssistant/ContentView.swift index d5355d1..fda47c3 100644 --- a/VoiceAssistant/ContentView.swift +++ b/VoiceAssistant/ContentView.swift @@ -87,7 +87,9 @@ struct DebugBar: View { @State private var localMusicFile: URL? var body: some View { + #if os(iOS) AudioSessionMonitorView() + #endif Toggle(isOn: $isVoiceProcessingEnabled) { Text("Voice processing enabled") @@ -152,10 +154,12 @@ struct DebugBar: View { player.volume = 1.0 player.play() + #if os(iOS) // Low volume workaround // https://developer.apple.com/forums/thread/721535 try AVAudioSession.sharedInstance().overrideOutputAudioPort(.speaker) try AVAudioSession.sharedInstance().overrideOutputAudioPort(.none) + #endif musicPlayerState = .playing self.player = player From 227797e036662452d3f0bce0e9ed0a84eb1adeee Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 19 Mar 2025 20:45:47 +0800 Subject: [PATCH 4/4] No Krisp for Catalyst --- VoiceAssistant/ContentView.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/VoiceAssistant/ContentView.swift b/VoiceAssistant/ContentView.swift index fda47c3..ffa7e00 100644 --- a/VoiceAssistant/ContentView.swift +++ b/VoiceAssistant/ContentView.swift @@ -1,7 +1,7 @@ import AVFAudio import LiveKit import SwiftUI -#if os(iOS) || os(macOS) +#if (os(iOS) && !targetEnvironment(macCatalyst)) || os(macOS) import LiveKitKrispNoiseFilter #endif @@ -10,12 +10,12 @@ struct ContentView: View { // Krisp is available only on iOS and macOS right now // Krisp is also a feature of LiveKit Cloud, so if you're using open-source / self-hosted you should remove this - #if os(iOS) || os(macOS) + #if (os(iOS) && !targetEnvironment(macCatalyst)) || os(macOS) private let krispProcessor = LiveKitKrispNoiseFilter() #endif init() { - #if os(iOS) || os(macOS) + #if (os(iOS) && !targetEnvironment(macCatalyst)) || os(macOS) AudioManager.shared.capturePostProcessingDelegate = krispProcessor #endif } @@ -32,7 +32,7 @@ struct ContentView: View { .padding() .environmentObject(room) .onAppear { - #if os(iOS) || os(macOS) + #if (os(iOS) && !targetEnvironment(macCatalyst)) || os(macOS) room.add(delegate: krispProcessor) #endif }