diff --git a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift index a7b168e..bc199f6 100755 --- a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift +++ b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift @@ -23,7 +23,8 @@ public enum PlaybackEndedReason: String { class AVPlayerWrapper: AVPlayerWrapperProtocol { // MARK: - Properties - fileprivate var avPlayer = AVPlayer() + internal var avPlayer = AVPlayer() + internal var audioTap: AudioTap? = nil private let playerObserver = AVPlayerObserver() internal let playerTimeObserver: AVPlayerTimeObserver private let playerItemNotificationObserver = AVPlayerItemNotificationObserver() @@ -69,7 +70,10 @@ class AVPlayerWrapper: AVPlayerWrapperProtocol { let currentState = self._state if (currentState != newValue) { self._state = newValue - self.delegate?.AVWrapper(didChangeState: newValue) + // the delegate can initiate a state change, resulting in a dealock in the getter. + DispatchQueue.main.async { + self.delegate?.AVWrapper(didChangeState: newValue) + } } } } @@ -385,6 +389,7 @@ class AVPlayerWrapper: AVPlayerWrapperProtocol { private func startObservingAVPlayer(item: AVPlayerItem) { playerItemObserver.startObserving(item: item) playerItemNotificationObserver.startObserving(item: item) + attachTap(audioTap, to: item) } private func stopObservingAVPlayerItem() { diff --git a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift index 0903339..0716eca 100755 --- a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift +++ b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift @@ -13,6 +13,8 @@ protocol AVPlayerWrapperProtocol: AnyObject { var state: AVPlayerWrapperState { get set } + var audioTap: AudioTap? { get set } + var playWhenReady: Bool { get set } var currentItem: AVPlayerItem? { get } diff --git a/Sources/SwiftAudioEx/AudioItem.swift b/Sources/SwiftAudioEx/AudioItem.swift index 833afe3..12e4946 100755 --- a/Sources/SwiftAudioEx/AudioItem.swift +++ b/Sources/SwiftAudioEx/AudioItem.swift @@ -29,7 +29,7 @@ public protocol AudioItem { func getAlbumTitle() -> String? func getSourceType() -> SourceType func getArtwork(_ handler: @escaping (AudioItemImage?) -> Void) - + func getArtworkURL() -> URL? } /// Make your `AudioItem`-subclass conform to this protocol to control which AVAudioTimePitchAlgorithm is used for each item. @@ -96,6 +96,9 @@ public class DefaultAudioItem: AudioItem { handler(artwork) } + public func getArtworkURL() -> URL? { + return nil + } } /// An AudioItem that also conforms to the `TimePitching`-protocol diff --git a/Sources/SwiftAudioEx/AudioPlayer.swift b/Sources/SwiftAudioEx/AudioPlayer.swift index 966c295..7c57d70 100755 --- a/Sources/SwiftAudioEx/AudioPlayer.swift +++ b/Sources/SwiftAudioEx/AudioPlayer.swift @@ -13,6 +13,14 @@ public typealias AudioPlayerState = AVPlayerWrapperState public class AudioPlayer: AVPlayerWrapperDelegate { /// The wrapper around the underlying AVPlayer let wrapper: AVPlayerWrapperProtocol = AVPlayerWrapper() + + /** + Set an instance of AudioTap, to receive frame information and audio buffer access during playback. + */ + public var audioTap: AudioTap? { + get { return wrapper.audioTap } + set(value) { wrapper.audioTap = value } + } public let nowPlayingInfoController: NowPlayingInfoControllerProtocol public let remoteCommandController: RemoteCommandController diff --git a/Sources/SwiftAudioEx/AudioTap.swift b/Sources/SwiftAudioEx/AudioTap.swift new file mode 100644 index 0000000..f350269 --- /dev/null +++ b/Sources/SwiftAudioEx/AudioTap.swift @@ -0,0 +1,98 @@ +// +// AudioTap.swift +// +// +// Created by Brandon Sneed on 3/31/24. +// + +import Foundation +import AVFoundation + +/** + Subclass this and set the AudioPlayer's `audioTap` property to start receiving the + audio stream. + */ +open class AudioTap { + // Called at tap initialization for a given player item. Use this to setup anything you might need. + open func initialize() { print("audioTap: initialize") } + // Called at teardown of the internal tap. Use this to reset any memory buffers you have created, etc. + open func finalize() { print("audioTap: finalize") } + // Called just before playback so you can perform setup based on the stream description. + open func prepare(description: AudioStreamBasicDescription) { print("audioTap: prepare") } + // Called just before finalize. + open func unprepare() { print("audioTap: unprepare") } + /** + Called periodically during audio stream playback. + + Example: + + ``` + func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) { + for channel in buffer { + // process audio samples here + //memset(channel.mData, 0, Int(channel.mDataByteSize)) + } + } + ``` + */ + open func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) { print("audioTap: process") } +} + +extension AVPlayerWrapper { + internal func attachTap(_ tap: AudioTap?, to item: AVPlayerItem) { + guard let tap else { return } + guard let track = item.asset.tracks(withMediaType: .audio).first else { + return + } + + let audioMix = AVMutableAudioMix() + let params = AVMutableAudioMixInputParameters(track: track) + + // we need to retain this pointer so it doesn't disappear out from under us. + // we'll then let it go after we finalize. If the tap changed upstream, we + // aren't going to pick up the new one until after this player item goes away. + let client = UnsafeMutableRawPointer(Unmanaged.passRetained(tap).toOpaque()) + + var callbacks = MTAudioProcessingTapCallbacks(version: kMTAudioProcessingTapCallbacksVersion_0, clientInfo: client) + { tapRef, clientInfo, tapStorageOut in + // initial tap setup + guard let clientInfo else { return } + tapStorageOut.pointee = clientInfo + let audioTap = Unmanaged.fromOpaque(clientInfo).takeUnretainedValue() + audioTap.initialize() + } finalize: { tapRef in + // clean up + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.finalize() + // we're done, we can let go of the pointer we retained. + Unmanaged.passUnretained(audioTap).release() + } prepare: { tapRef, maxFrames, processingFormat in + // allocate memory for sound processing + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.prepare(description: processingFormat.pointee) + } unprepare: { tapRef in + // deallocate memory for sound processing + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.unprepare() + } process: { tapRef, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut in + guard noErr == MTAudioProcessingTapGetSourceAudio(tapRef, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut) else { + return + } + + // process sound data + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.process(numberOfFrames: numberFrames, buffer: UnsafeMutableAudioBufferListPointer(bufferListInOut)) + } + + var tapRef: Unmanaged? + let error = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PreEffects, &tapRef) + assert(error == noErr) + + params.audioTapProcessor = tapRef?.takeUnretainedValue() + tapRef?.release() + + audioMix.inputParameters = [params] + item.audioMix = audioMix + } +} + diff --git a/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift b/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift index f46e3d3..4b93d0b 100644 --- a/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift +++ b/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift @@ -63,6 +63,7 @@ class AVPlayerItemObserver: NSObject { self.isObserving = true self.observingItem = item + item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.duration, options: [.new], context: &AVPlayerItemObserver.context) item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.loadedTimeRanges, options: [.new], context: &AVPlayerItemObserver.context) item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.playbackLikelyToKeepUp, options: [.new], context: &AVPlayerItemObserver.context) @@ -79,6 +80,9 @@ class AVPlayerItemObserver: NSObject { return } + // BKS: remove a tap if we had one. + observingItem.audioMix = nil + observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.duration, context: &AVPlayerItemObserver.context) observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.loadedTimeRanges, context: &AVPlayerItemObserver.context) observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.playbackLikelyToKeepUp, context: &AVPlayerItemObserver.context) diff --git a/Sources/SwiftAudioEx/Utils/Devices.swift b/Sources/SwiftAudioEx/Utils/Devices.swift new file mode 100644 index 0000000..6e43907 --- /dev/null +++ b/Sources/SwiftAudioEx/Utils/Devices.swift @@ -0,0 +1,204 @@ +// +// File.swift +// +// +// Created by Brandon Sneed on 4/1/24. +// + +import Foundation +import AVFoundation +import CoreAudio + +public class AudioDevice: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + return name ?? "Unknown" + } + + public var debugDescription: String { + return name ?? "Unknown" + } + + static var system: AudioDevice = { + return AudioDevice() + }() + + public let deviceID: AudioDeviceID? + public let uniqueID: String? + public let name: String? + + internal init(deviceID: AudioDeviceID) { + self.deviceID = deviceID + self.uniqueID = Self.propertyValue(deviceID: deviceID, selector: AudioObjectPropertySelector(kAudioDevicePropertyDeviceUID)) + self.name = Self.propertyValue(deviceID: deviceID, selector: AudioObjectPropertySelector(kAudioDevicePropertyDeviceNameCFString)) + } + + internal init() { + self.deviceID = 0 + self.uniqueID = nil + self.name = "System" + } +} + +extension AudioDevice { + static func hasOutput(deviceID: AudioDeviceID) -> Bool { + var status: OSStatus = 0 + var address = AudioObjectPropertyAddress( + mSelector: AudioObjectPropertySelector(kAudioDevicePropertyStreamConfiguration), + mScope: AudioObjectPropertyScope(kAudioObjectPropertyScopeOutput), + mElement: 0) + + var size: UInt32 = 0 + withUnsafeMutablePointer(to: &size) { size in + withUnsafePointer(to: &address) { addressPtr in + status = AudioObjectGetPropertyDataSize(deviceID, addressPtr, 0, nil, size) + } + } + + if status != 0 { + // we weren't able to get the size + return false + } + + let bufferList = UnsafeMutablePointer.allocate(capacity: Int(size)) + withUnsafeMutablePointer(to: &size) { size in + withUnsafePointer(to: &address) { addressPtr in + status = AudioObjectGetPropertyData(deviceID, addressPtr, 0, nil, size, bufferList) + } + } + + if status != 0 { + // we couldn't get the buffer list + return false + } + + let buffers = UnsafeMutableAudioBufferListPointer(bufferList) + for buffer in buffers { + if buffer.mNumberChannels > 0 { + return true + } + } + + return false + } + + static internal func propertyValue(deviceID: AudioDeviceID, selector: AudioObjectPropertySelector) -> String? { + var result: String? = nil + + var address = AudioObjectPropertyAddress( + mSelector: selector, + mScope: AudioObjectPropertyScope(kAudioObjectPropertyScopeGlobal), + mElement: AudioObjectPropertyElement(kAudioObjectPropertyElementMain)) + + var name: Unmanaged? + var size = UInt32(MemoryLayout.size) + withUnsafeMutablePointer(to: &size) { size in + withUnsafePointer(to: &address) { addressPtr in + let status = AudioObjectGetPropertyData(deviceID, addressPtr, 0, nil, size, &name) + if status != 0 { + return + } + result = name?.takeUnretainedValue() as String? + } + } + + return result + } +} + +extension AudioPlayer { + /** + Set the output device for the Player. Default is system. + */ + public func setOutputDevice(_ device: AudioDevice) { + guard let wrapper = wrapper as? AVPlayerWrapper else { return } + wrapper.avPlayer.audioOutputDeviceUniqueID = device.uniqueID + } + + /** + Get the current output device + */ + public var outputDevice: AudioDevice { + get { + guard let wrapper = wrapper as? AVPlayerWrapper else { return AudioDevice.system } + guard let uniqueID = wrapper.avPlayer.audioOutputDeviceUniqueID else { return AudioDevice.system } + let devices = localDevices.filter { device in + return device.uniqueID == uniqueID + } + if let match = devices.first { + return match + } + return AudioDevice.system + } + set(value) { + guard let wrapper = wrapper as? AVPlayerWrapper else { return } + wrapper.avPlayer.audioOutputDeviceUniqueID = value.uniqueID + } + } + + /** + Get a list of local audio devices capable of output. + + This list will *NOT* include AirPlay devices. For Airplay and other streaming + audio devices, see AVRoutePickerView. + */ + public var localDevices: [AudioDevice] { + get { + var status: OSStatus = 0 + var address = AudioObjectPropertyAddress( + mSelector: AudioObjectPropertySelector(kAudioHardwarePropertyDevices), + mScope: AudioObjectPropertyScope(kAudioObjectPropertyScopeGlobal), + mElement: AudioObjectPropertyElement(kAudioObjectPropertyElementMaster)) + + var size: UInt32 = 0 + withUnsafeMutablePointer(to: &size) { size in + withUnsafePointer(to: &address) { address in + status = AudioObjectGetPropertyDataSize( + AudioObjectID(kAudioObjectSystemObject), + address, + UInt32(MemoryLayout.size), + nil, + size) + } + } + + if status != 0 { + // we couldn't get a data size + return [] + } + + let deviceCount = size / UInt32(MemoryLayout.size) + var deviceIDs = [AudioDeviceID]() + for _ in 0.. 4 { + // swap it out part-way through the first track. + self.audioPlayer.audioTap = DummyAudioTap(tapIndex: 2) + } + } + + audioPlayer.audioTap = DummyAudioTap(tapIndex: 1) + audioPlayer.load(item: FiveSecondSource.getAudioItem()) + audioPlayer.play() + + RunLoop.current.run(until: Date(timeIntervalSinceNow: 6)) + + audioPlayer.load(item: FiveSecondSource.getAudioItem()) + audioPlayer.play() + + RunLoop.current.run(until: Date(timeIntervalSinceNow: 6)) + + let tap1Active = DummyAudioTap.outputs.contains { output in + return output.contains("audioTap 1: process") + } + + let tap2Active = DummyAudioTap.outputs.contains { output in + return output.contains("audioTap 2: process") + } + XCTAssertTrue(tap1Active) + XCTAssertTrue(tap2Active) + } + + // MARK: - Device Tests + + func testAudioDeviceListing() { + // I know this test kind of stinks. Devices will vary on every system, + // and i can't really test device output in CI. :/ + let list = audioPlayer.localDevices + print(list) + } + // MARK: - Failure func testFailEventOnLoadWithNonMalformedURL() { diff --git a/Tests/SwiftAudioExTests/Mocks/DummyAudioTap.swift b/Tests/SwiftAudioExTests/Mocks/DummyAudioTap.swift new file mode 100644 index 0000000..e952e62 --- /dev/null +++ b/Tests/SwiftAudioExTests/Mocks/DummyAudioTap.swift @@ -0,0 +1,40 @@ +// +// File.swift +// +// +// Created by Brandon Sneed on 4/1/24. +// + +import Foundation +import CoreAudio +@testable import SwiftAudioEx + +class DummyAudioTap: AudioTap { + static var outputs = [String]() + + let tapIndex: Int + + init(tapIndex: Int) { + self.tapIndex = tapIndex + } + + override func initialize() { + Self.outputs.append("audioTap \(tapIndex): initialize") + } + + override func finalize() { + Self.outputs.append("audioTap \(tapIndex): finalize") + } + + override func prepare(description: AudioStreamBasicDescription) { + Self.outputs.append("audioTap \(tapIndex): prepare") + } + + override func unprepare() { + Self.outputs.append("audioTap \(tapIndex): unprepare") + } + + override func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) { + Self.outputs.append("audioTap \(tapIndex): process") + } +} diff --git a/Tests/SwiftAudioExTests/NowPlayingInfoTests.swift b/Tests/SwiftAudioExTests/NowPlayingInfoTests.swift index 075458e..db6ae8a 100644 --- a/Tests/SwiftAudioExTests/NowPlayingInfoTests.swift +++ b/Tests/SwiftAudioExTests/NowPlayingInfoTests.swift @@ -33,7 +33,12 @@ class NowPlayingInfoTests: XCTestCase { func testNowPlayingInfoControllerPlaybackValuesUpdate() { let item = LongSource.getAudioItem() + + // State has become somewhat async to prevent a deadlock, + // so this isn't instantaneous anymore and needs a teensy bit of time. audioPlayer.load(item: item, playWhenReady: true) + + RunLoop.current.run(until: Date(timeIntervalSinceNow: 1)) XCTAssertNotNil(nowPlayingController.getRate()) XCTAssertNotNil(nowPlayingController.getDuration())