From 3c17effbbafe1656a3269d59789ec5e4c292ebd8 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Mar 2022 03:24:03 +0900 Subject: [PATCH 01/24] first attempt --- sdk/BUILD.gn | 68 ++++++++++- .../RTCAudioDeviceModule+Private.h | 22 ++++ .../api/peerconnection/RTCAudioDeviceModule.h | 40 +++++++ .../peerconnection/RTCAudioDeviceModule.mm | 110 ++++++++++++++++++ .../api/peerconnection/RTCDevice+Private.h | 22 ++++ sdk/objc/api/peerconnection/RTCDevice.h | 27 +++++ sdk/objc/api/peerconnection/RTCDevice.mm | 28 +++++ .../peerconnection/RTCPeerConnectionFactory.h | 3 + .../RTCPeerConnectionFactory.mm | 16 ++- sdk/objc/native/api/audio_device_module.mm | 4 + 10 files changed, 338 insertions(+), 2 deletions(-) create mode 100644 sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h create mode 100644 sdk/objc/api/peerconnection/RTCAudioDeviceModule.h create mode 100644 sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm create mode 100644 sdk/objc/api/peerconnection/RTCDevice+Private.h create mode 100644 sdk/objc/api/peerconnection/RTCDevice.h create mode 100644 sdk/objc/api/peerconnection/RTCDevice.mm diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 672b4f5b4a..1463801828 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -374,6 +374,62 @@ if (is_ios || is_mac) { } } + if (is_mac) { + rtc_library("native_api_audio_device_module") { + visibility = [ "*" ] + + sources = [ + "objc/native/api/audio_device_module.h", + "objc/native/api/audio_device_module.mm", + ] + + deps = [ + ":audio_device", + "../modules/audio_device:audio_device_api", + "../modules/audio_device:audio_device_generic", + "../rtc_base:checks", + "../rtc_base:rtc_base_approved", + "../system_wrappers", + ] + } + + rtc_source_set("audio_session_observer") { + visibility = [ ":*" ] + + sources = [ "objc/native/src/audio/audio_session_observer.h" ] + + deps = [ + "../rtc_base", + "../rtc_base:threading", + ] + } + + rtc_library("audio_device") { + visibility = [ "*" ] + + sources = [ + "objc/native/src/audio/helpers.h", + "objc/native/src/audio/helpers.mm", + ] + + deps = [ + ":base_objc", + "../api:array_view", + "../modules/audio_device:audio_device_api", + "../modules/audio_device:audio_device_buffer", + "../modules/audio_device:audio_device_generic", + "../rtc_base", + "../rtc_base:checks", + "../rtc_base:threading", + "../system_wrappers:field_trial", + "../system_wrappers:metrics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] + + frameworks = [ "AudioToolbox.framework" ] + } + } + rtc_library("videosource_objc") { sources = [ "objc/api/peerconnection/RTCVideoSource+Private.h", @@ -882,6 +938,12 @@ if (is_ios || is_mac) { "..:no_global_constructors", ] sources = [ + "objc/api/peerconnection/RTCAudioDeviceModule.h", + "objc/api/peerconnection/RTCAudioDeviceModule+Private.h", + "objc/api/peerconnection/RTCAudioDeviceModule.mm", + "objc/api/peerconnection/RTCDevice.h", + "objc/api/peerconnection/RTCDevice+Private.h", + "objc/api/peerconnection/RTCDevice.mm", "objc/api/peerconnection/RTCAudioSource+Private.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioSource.mm", @@ -1034,7 +1096,7 @@ if (is_ios || is_mac) { "../system_wrappers:metrics", ] - if (is_ios) { + if (is_ios || is_mac) { deps += [ ":native_api_audio_device_module" ] } } @@ -1273,6 +1335,8 @@ if (is_ios || is_mac) { "objc/helpers/RTCCameraPreviewView.h", "objc/helpers/RTCDispatcher.h", "objc/helpers/UIDevice+RTCDevice.h", + "objc/api/peerconnection/RTCAudioDeviceModule.h", + "objc/api/peerconnection/RTCDevice.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioTrack.h", "objc/api/peerconnection/RTCConfiguration.h", @@ -1387,6 +1451,8 @@ if (is_ios || is_mac) { output_name = "WebRTC" sources = [ + "objc/api/peerconnection/RTCAudioDeviceModule.h", + "objc/api/peerconnection/RTCDevice.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioTrack.h", "objc/api/peerconnection/RTCCertificate.h", diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h new file mode 100644 index 0000000000..f4a8ff289c --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h @@ -0,0 +1,22 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioDeviceModule.h" +#import "sdk/objc/native/api/audio_device_module.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCAudioDeviceModule () + +- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h new file mode 100644 index 0000000000..0569201402 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -0,0 +1,40 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#import "RTCMacros.h" +#import "RTCDevice.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioDeviceModule) : NSObject + +- (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer; + +@property(nonatomic, readonly) NSArray *playoutDevices; +@property(nonatomic, readonly) NSArray *recordingDevices; + +@property(nonatomic, readonly) BOOL playing; +@property(nonatomic, readonly) BOOL recording; + +- (BOOL)setPlayoutDevice:(uint16_t) index; +- (BOOL)startPlayout; +- (BOOL)stopPlayout; + +- (BOOL)setRecordingDevice:(uint16_t) index; +- (BOOL)startRecording; +- (BOOL)stopRecording; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm new file mode 100644 index 0000000000..58a40d2c78 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -0,0 +1,110 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#import "RTCAudioDeviceModule+Private.h" +#import "RTCDevice+Private.h" + +#include "rtc_base/ref_counted_object.h" +#import "sdk/objc/native/api/audio_device_module.h" + +@implementation RTCAudioDeviceModule { + rtc::scoped_refptr _nativeModule; +} + +- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module { + self = [super init]; + _nativeModule = module; + return self; +} + +- (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { + // TODO: Implement + // _nativeModule->CaptureSampleBuffer(sampleBuffer); +} + +- (NSArray *)playoutDevices { + + char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; + char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; + + NSMutableArray *result = [NSMutableArray array]; + + int16_t count = _nativeModule->PlayoutDevices(); + + if (count > 0) { + for (int i = 0; i < count; i++) { + _nativeModule->PlayoutDeviceName(i, name, guid); + NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; + NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; + RTCDevice *device = [[RTCDevice alloc] initWithGUID:strGUID name:strName]; + [result addObject: device]; + } + } + + return result; +} + +- (NSArray *)recordingDevices { + + char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; + char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; + + NSMutableArray *result = [NSMutableArray array]; + + int16_t count = _nativeModule->RecordingDevices(); + + if (count > 0) { + for (int i = 0; i < count; i++) { + _nativeModule->RecordingDeviceName(i, name, guid); + NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; + NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; + RTCDevice *device = [[RTCDevice alloc] initWithGUID:strGUID name:strName]; + [result addObject: device]; + } + } + + return result; +} + +- (BOOL)playing { + return _nativeModule->Playing(); +} + +- (BOOL)setPlayoutDevice:(uint16_t) index { + return _nativeModule->SetPlayoutDevice(index) == 0; +} + +- (BOOL)startPlayout { + return _nativeModule->StartPlayout() == 0; +} + +- (BOOL)stopPlayout { + return _nativeModule->StopPlayout() == 0; +} + +- (BOOL)recording { + return _nativeModule->Recording(); +} + +- (BOOL)setRecordingDevice:(uint16_t) index { + return _nativeModule->SetRecordingDevice(index) == 0; +} + +- (BOOL)startRecording { + return _nativeModule->StartRecording() == 0; +} + +- (BOOL)stopRecording { + return _nativeModule->StopRecording() == 0; +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCDevice+Private.h b/sdk/objc/api/peerconnection/RTCDevice+Private.h new file mode 100644 index 0000000000..47ac702e05 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCDevice+Private.h @@ -0,0 +1,22 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDevice.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCDevice () + +- (instancetype)initWithGUID:(NSString *)guid + name:(NSString* )name; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCDevice.h b/sdk/objc/api/peerconnection/RTCDevice.h new file mode 100644 index 0000000000..d6b6bc5363 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCDevice.h @@ -0,0 +1,27 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTCDevice : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@property(nonatomic, copy, readonly) NSString *guid; +@property(nonatomic, copy, readonly) NSString *name; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCDevice.mm b/sdk/objc/api/peerconnection/RTCDevice.mm new file mode 100644 index 0000000000..98beaf778d --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCDevice.mm @@ -0,0 +1,28 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDevice+Private.h" +#include "rtc_base/ref_counted_object.h" + +@implementation RTCDevice + +@synthesize guid = _guid; +@synthesize name = _name; + +- (instancetype)initWithGUID:(NSString *)guid + name:(NSString* )name { + if (self = [super init]) { + _guid = guid; + _name = name; + } + return self; +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h index a566e823ce..b92e62ea90 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h @@ -11,6 +11,7 @@ #import #import "RTCMacros.h" +#import "RTCAudioDeviceModule.h" NS_ASSUME_NONNULL_BEGIN @@ -48,6 +49,8 @@ RTC_OBJC_EXPORT decoderFactory: (nullable id)decoderFactory; +@property(nonatomic, readonly) RTCAudioDeviceModule *audioDeviceModule; + /** Initialize an RTCAudioSource with constraints. */ - (RTC_OBJC_TYPE(RTCAudioSource) *)audioSourceWithConstraints: (nullable RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints; diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index da94078152..dae3752bda 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -14,6 +14,9 @@ #import "RTCPeerConnectionFactory+Private.h" #import "RTCPeerConnectionFactoryOptions+Private.h" +#import "RTCAudioDeviceModule.h" +#import "RTCAudioDeviceModule+Private.h" + #import "RTCAudioSource+Private.h" #import "RTCAudioTrack+Private.h" #import "RTCMediaConstraints+Private.h" @@ -63,6 +66,8 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { std::unique_ptr _networkThread; std::unique_ptr _workerThread; std::unique_ptr _signalingThread; + rtc::scoped_refptr _audioDeviceModule; + BOOL _hasStartedAecDump; } @@ -76,6 +81,10 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { #endif } +- (RTCAudioDeviceModule *)audioDeviceModule { + return [[RTCAudioDeviceModule alloc] initWithNativeModule: _audioDeviceModule]; +} + - (instancetype)init { #ifdef HAVE_NO_MEDIA return [self initWithNoMedia]; @@ -220,7 +229,12 @@ - (instancetype)initWithNativeAudioEncoderFactory: dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); dependencies.trials = std::make_unique(); cricket::MediaEngineDependencies media_deps; - media_deps.adm = std::move(audioDeviceModule); + + rtc::scoped_refptr adm = webrtc::AudioDeviceModule::Create(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, dependencies.task_queue_factory.get()); + adm->Init(); + _audioDeviceModule = std::move(adm); + + media_deps.adm = std::move(adm); media_deps.task_queue_factory = dependencies.task_queue_factory.get(); media_deps.audio_encoder_factory = std::move(audioEncoderFactory); media_deps.audio_decoder_factory = std::move(audioDecoderFactory); diff --git a/sdk/objc/native/api/audio_device_module.mm b/sdk/objc/native/api/audio_device_module.mm index 3c2790e38d..b138445990 100644 --- a/sdk/objc/native/api/audio_device_module.mm +++ b/sdk/objc/native/api/audio_device_module.mm @@ -13,7 +13,11 @@ #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" +#if defined(WEBRTC_IOS) #include "sdk/objc/native/src/audio/audio_device_module_ios.h" +#endif + +#include "modules/audio_device/include/audio_device.h" namespace webrtc { From 42029a710cf29e3dd99932c0ad709e0a18196e86 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Mar 2022 03:37:05 +0900 Subject: [PATCH 02/24] remove unused dep --- sdk/BUILD.gn | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 1463801828..395e57484b 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -393,17 +393,6 @@ if (is_ios || is_mac) { ] } - rtc_source_set("audio_session_observer") { - visibility = [ ":*" ] - - sources = [ "objc/native/src/audio/audio_session_observer.h" ] - - deps = [ - "../rtc_base", - "../rtc_base:threading", - ] - } - rtc_library("audio_device") { visibility = [ "*" ] From 4c929eb548acc30e54dd995149cf0cda4a28998f Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Mar 2022 17:42:02 +0900 Subject: [PATCH 03/24] init playout / recording --- sdk/objc/api/peerconnection/RTCAudioDeviceModule.h | 2 ++ sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index 0569201402..10606d1d5c 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -30,10 +30,12 @@ RTC_OBJC_EXPORT - (BOOL)setPlayoutDevice:(uint16_t) index; - (BOOL)startPlayout; - (BOOL)stopPlayout; +- (BOOL)initPlayout; - (BOOL)setRecordingDevice:(uint16_t) index; - (BOOL)startRecording; - (BOOL)stopRecording; +- (BOOL)initRecording; @end diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 58a40d2c78..8b79c42ed9 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -91,6 +91,10 @@ - (BOOL)stopPlayout { return _nativeModule->StopPlayout() == 0; } +- (BOOL)initPlayout { + return _nativeModule->InitPlayout() == 0; +} + - (BOOL)recording { return _nativeModule->Recording(); } @@ -107,4 +111,8 @@ - (BOOL)stopRecording { return _nativeModule->StopRecording() == 0; } +- (BOOL)initRecording { + return _nativeModule->InitRecording() == 0; +} + @end From efc3c323b099db4123d3497b6365699d20ec6570 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Mar 2022 23:20:54 +0900 Subject: [PATCH 04/24] use AudioDeviceID as guid --- modules/audio_device/mac/audio_device_mac.cc | 22 +++++++++----------- modules/audio_device/mac/audio_device_mac.h | 3 ++- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index e0d4419c81..4a24522fff 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -849,12 +849,9 @@ int32_t AudioDeviceMac::PlayoutDeviceName(uint16_t index, } memset(name, 0, kAdmMaxDeviceNameSize); + memset(guid, 0, kAdmMaxGuidSize); - if (guid != NULL) { - memset(guid, 0, kAdmMaxGuidSize); - } - - return GetDeviceName(kAudioDevicePropertyScopeOutput, index, name); + return GetDeviceName(kAudioDevicePropertyScopeOutput, index, name, guid); } int32_t AudioDeviceMac::RecordingDeviceName(uint16_t index, @@ -867,12 +864,9 @@ int32_t AudioDeviceMac::RecordingDeviceName(uint16_t index, } memset(name, 0, kAdmMaxDeviceNameSize); - - if (guid != NULL) { - memset(guid, 0, kAdmMaxGuidSize); - } - - return GetDeviceName(kAudioDevicePropertyScopeInput, index, name); + memset(guid, 0, kAdmMaxGuidSize); + + return GetDeviceName(kAudioDevicePropertyScopeInput, index, name, guid); } int16_t AudioDeviceMac::RecordingDevices() { @@ -1665,7 +1659,8 @@ int32_t AudioDeviceMac::GetNumberDevices(const AudioObjectPropertyScope scope, int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, const uint16_t index, - char* name) { + char* name, + char* guid) { OSStatus err = noErr; UInt32 len = kAdmMaxDeviceNameSize; AudioDeviceID deviceIds[MaxNumberDevices]; @@ -1704,6 +1699,9 @@ int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, } } + std::string strData = std::to_string(deviceIds[index]); + strcpy(guid, strData.c_str()); + AudioObjectPropertyAddress propertyAddress = {kAudioDevicePropertyDeviceName, scope, 0}; diff --git a/modules/audio_device/mac/audio_device_mac.h b/modules/audio_device/mac/audio_device_mac.h index f9504b64b5..241714b851 100644 --- a/modules/audio_device/mac/audio_device_mac.h +++ b/modules/audio_device/mac/audio_device_mac.h @@ -178,7 +178,8 @@ class AudioDeviceMac : public AudioDeviceGeneric { int32_t GetDeviceName(const AudioObjectPropertyScope scope, const uint16_t index, - char* name); + char* name, + char* guid); int32_t InitDevice(uint16_t userDeviceIndex, AudioDeviceID& deviceId, From a5042dcc3c9ed1e85cdea7cccd6fe6b8d226fcc8 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 12 Mar 2022 00:41:35 +0900 Subject: [PATCH 05/24] switch device method --- .../api/peerconnection/RTCAudioDeviceModule.h | 6 ++ .../peerconnection/RTCAudioDeviceModule.mm | 94 +++++++++++++++---- 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index 10606d1d5c..f65a165bdc 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -27,11 +27,17 @@ RTC_OBJC_EXPORT @property(nonatomic, readonly) BOOL playing; @property(nonatomic, readonly) BOOL recording; +// Executes low-level API's in sequence to switch the device +- (BOOL)switchPlayoutDevice: (nullable RTCDevice *)device; +- (BOOL)switchRecordingDevice: (nullable RTCDevice *)device; + +// Low-level APIs - (BOOL)setPlayoutDevice:(uint16_t) index; - (BOOL)startPlayout; - (BOOL)stopPlayout; - (BOOL)initPlayout; +// Low-level APIs - (BOOL)setRecordingDevice:(uint16_t) index; - (BOOL)startRecording; - (BOOL)stopRecording; diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 8b79c42ed9..dbed6168be 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -17,18 +17,18 @@ #import "sdk/objc/native/api/audio_device_module.h" @implementation RTCAudioDeviceModule { - rtc::scoped_refptr _nativeModule; + rtc::scoped_refptr _native; } - (instancetype)initWithNativeModule:(rtc::scoped_refptr )module { self = [super init]; - _nativeModule = module; + _native = module; return self; } - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { // TODO: Implement - // _nativeModule->CaptureSampleBuffer(sampleBuffer); + // _native->CaptureSampleBuffer(sampleBuffer); } - (NSArray *)playoutDevices { @@ -38,11 +38,11 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { NSMutableArray *result = [NSMutableArray array]; - int16_t count = _nativeModule->PlayoutDevices(); + int16_t count = _native->PlayoutDevices(); if (count > 0) { for (int i = 0; i < count; i++) { - _nativeModule->PlayoutDeviceName(i, name, guid); + _native->PlayoutDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; RTCDevice *device = [[RTCDevice alloc] initWithGUID:strGUID name:strName]; @@ -60,11 +60,11 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { NSMutableArray *result = [NSMutableArray array]; - int16_t count = _nativeModule->RecordingDevices(); + int16_t count = _native->RecordingDevices(); if (count > 0) { for (int i = 0; i < count; i++) { - _nativeModule->RecordingDeviceName(i, name, guid); + _native->RecordingDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; RTCDevice *device = [[RTCDevice alloc] initWithGUID:strGUID name:strName]; @@ -75,44 +75,104 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { return result; } +- (BOOL)switchPlayoutDevice: (nullable RTCDevice *)device { + + NSUInteger index = 0; + NSArray *devices = [self playoutDevices]; + + if ([devices count] == 0) { + return NO; + } + + if (device != nil) { + index = [devices indexOfObjectPassingTest:^BOOL(RTCDevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.guid isEqualToString:device.guid]); + }]; + if (index == NSNotFound) { + return NO; + } + } + + _native->StopPlayout(); + + if (_native->SetPlayoutDevice(index) == 0 + && _native->InitPlayout() == 0 + && _native->StartPlayout() == 0) { + + return YES; + } + + return NO; +} + +- (BOOL)switchRecordingDevice: (nullable RTCDevice *)device { + + NSUInteger index = 0; + NSArray *devices = [self recordingDevices]; + + if ([devices count] == 0) { + return NO; + } + + if (device != nil) { + index = [devices indexOfObjectPassingTest:^BOOL(RTCDevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.guid isEqualToString:device.guid]); + }]; + if (index == NSNotFound) { + return NO; + } + } + + _native->StopRecording(); + + if (_native->SetRecordingDevice(index) == 0 + && _native->InitRecording() == 0 + && _native->StartRecording() == 0) { + + return YES; + } + + return NO; +} + - (BOOL)playing { - return _nativeModule->Playing(); + return _native->Playing(); } - (BOOL)setPlayoutDevice:(uint16_t) index { - return _nativeModule->SetPlayoutDevice(index) == 0; + return _native->SetPlayoutDevice(index) == 0; } - (BOOL)startPlayout { - return _nativeModule->StartPlayout() == 0; + return _native->StartPlayout() == 0; } - (BOOL)stopPlayout { - return _nativeModule->StopPlayout() == 0; + return _native->StopPlayout() == 0; } - (BOOL)initPlayout { - return _nativeModule->InitPlayout() == 0; + return _native->InitPlayout() == 0; } - (BOOL)recording { - return _nativeModule->Recording(); + return _native->Recording(); } - (BOOL)setRecordingDevice:(uint16_t) index { - return _nativeModule->SetRecordingDevice(index) == 0; + return _native->SetRecordingDevice(index) == 0; } - (BOOL)startRecording { - return _nativeModule->StartRecording() == 0; + return _native->StartRecording() == 0; } - (BOOL)stopRecording { - return _nativeModule->StopRecording() == 0; + return _native->StopRecording() == 0; } - (BOOL)initRecording { - return _nativeModule->InitRecording() == 0; + return _native->InitRecording() == 0; } @end From dfd4ace8ffd9e082b5df0b82daf1385c3f5d80e2 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 12 Mar 2022 00:42:22 +0900 Subject: [PATCH 06/24] equality --- sdk/objc/api/peerconnection/RTCDevice.mm | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sdk/objc/api/peerconnection/RTCDevice.mm b/sdk/objc/api/peerconnection/RTCDevice.mm index 98beaf778d..36fb43359b 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.mm +++ b/sdk/objc/api/peerconnection/RTCDevice.mm @@ -25,4 +25,23 @@ - (instancetype)initWithGUID:(NSString *)guid return self; } + +- (BOOL)isEqual:(id)object { + if (self == object) { + return YES; + } + if (object == nil) { + return NO; + } + if (![object isMemberOfClass:[self class]]) { + return NO; + } + + return [_guid isEqualToString:((RTC_OBJC_TYPE(RTCDevice) *)object).guid]; +} + +- (NSUInteger)hash { + return [_guid hash]; +} + @end From eda19e11a12b9307229b03b5c513025e33267bfa Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 12 Mar 2022 02:58:33 +0900 Subject: [PATCH 07/24] default device --- sdk/objc/api/peerconnection/RTCDevice.h | 1 + sdk/objc/api/peerconnection/RTCDevice.mm | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/sdk/objc/api/peerconnection/RTCDevice.h b/sdk/objc/api/peerconnection/RTCDevice.h index d6b6bc5363..40a22a695c 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.h +++ b/sdk/objc/api/peerconnection/RTCDevice.h @@ -17,6 +17,7 @@ NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @interface RTCDevice : NSObject ++ (instancetype)defaultDevice; - (instancetype)init NS_UNAVAILABLE; @property(nonatomic, copy, readonly) NSString *guid; diff --git a/sdk/objc/api/peerconnection/RTCDevice.mm b/sdk/objc/api/peerconnection/RTCDevice.mm index 36fb43359b..64447f2161 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.mm +++ b/sdk/objc/api/peerconnection/RTCDevice.mm @@ -11,11 +11,18 @@ #import "RTCDevice+Private.h" #include "rtc_base/ref_counted_object.h" +NSString *const kDefaultDeviceId = @"default"; + @implementation RTCDevice @synthesize guid = _guid; @synthesize name = _name; ++ (instancetype)defaultDevice { + return [[self alloc] initWithGUID: kDefaultDeviceId + name: @""]; +} + - (instancetype)initWithGUID:(NSString *)guid name:(NSString* )name { if (self = [super init]) { From 507913cf9a8a54da43cb36c162f2c4a08e35b793 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 13 Mar 2022 02:32:37 +0900 Subject: [PATCH 08/24] `isDefault` property --- sdk/objc/api/peerconnection/RTCDevice.h | 1 + sdk/objc/api/peerconnection/RTCDevice.mm | 3 +++ 2 files changed, 4 insertions(+) diff --git a/sdk/objc/api/peerconnection/RTCDevice.h b/sdk/objc/api/peerconnection/RTCDevice.h index 40a22a695c..ff6c446fb9 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.h +++ b/sdk/objc/api/peerconnection/RTCDevice.h @@ -20,6 +20,7 @@ RTC_OBJC_EXPORT + (instancetype)defaultDevice; - (instancetype)init NS_UNAVAILABLE; +@property(nonatomic, readonly) BOOL isDefault; @property(nonatomic, copy, readonly) NSString *guid; @property(nonatomic, copy, readonly) NSString *name; diff --git a/sdk/objc/api/peerconnection/RTCDevice.mm b/sdk/objc/api/peerconnection/RTCDevice.mm index 64447f2161..1594b0e029 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.mm +++ b/sdk/objc/api/peerconnection/RTCDevice.mm @@ -32,6 +32,9 @@ - (instancetype)initWithGUID:(NSString *)guid return self; } +- (BOOL)isDefault { + return [_guid isEqualToString: kDefaultDeviceId]; +} - (BOOL)isEqual:(id)object { if (self == object) { From c08e4cfa6b4882552a4f0d87b0261912fd49986b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 13 Mar 2022 03:02:16 +0900 Subject: [PATCH 09/24] dont format default device name --- modules/audio_device/mac/audio_device_mac.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index 4a24522fff..64ee58b9df 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -1699,20 +1699,17 @@ int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, } } - std::string strData = std::to_string(deviceIds[index]); + std::string strData; + if (isDefaultDevice) { + strData = "default"; + } else { + strData = std::to_string(deviceIds[index]); + } strcpy(guid, strData.c_str()); AudioObjectPropertyAddress propertyAddress = {kAudioDevicePropertyDeviceName, scope, 0}; - if (isDefaultDevice) { - char devName[len]; - - WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData(usedID, &propertyAddress, - 0, NULL, &len, devName)); - - sprintf(name, "default (%s)", devName); - } else { if (index < numberDevices) { usedID = deviceIds[index]; } else { @@ -1721,7 +1718,6 @@ int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData(usedID, &propertyAddress, 0, NULL, &len, name)); - } return 0; } From e063d9366d7aa416d0eada6379b5e9d71a405a46 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 13 Mar 2022 03:03:03 +0900 Subject: [PATCH 10/24] type param --- sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm | 4 ++-- sdk/objc/api/peerconnection/RTCDevice+Private.h | 3 ++- sdk/objc/api/peerconnection/RTCDevice.h | 11 ++++++++++- sdk/objc/api/peerconnection/RTCDevice.mm | 10 +++++++--- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index dbed6168be..289d866275 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -45,7 +45,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { _native->PlayoutDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCDevice *device = [[RTCDevice alloc] initWithGUID:strGUID name:strName]; + RTCDevice *device = [[RTCDevice alloc] initWithType:RTCDeviceTypeOutput guid:strGUID name:strName]; [result addObject: device]; } } @@ -67,7 +67,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { _native->RecordingDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCDevice *device = [[RTCDevice alloc] initWithGUID:strGUID name:strName]; + RTCDevice *device = [[RTCDevice alloc] initWithType:RTCDeviceTypeInput guid:strGUID name:strName]; [result addObject: device]; } } diff --git a/sdk/objc/api/peerconnection/RTCDevice+Private.h b/sdk/objc/api/peerconnection/RTCDevice+Private.h index 47ac702e05..cc41bc0454 100644 --- a/sdk/objc/api/peerconnection/RTCDevice+Private.h +++ b/sdk/objc/api/peerconnection/RTCDevice+Private.h @@ -14,7 +14,8 @@ NS_ASSUME_NONNULL_BEGIN @interface RTCDevice () -- (instancetype)initWithGUID:(NSString *)guid +- (instancetype)initWithType:(RTCDeviceType)type + guid:(NSString *)guid name:(NSString* )name; @end diff --git a/sdk/objc/api/peerconnection/RTCDevice.h b/sdk/objc/api/peerconnection/RTCDevice.h index ff6c446fb9..51439441fa 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.h +++ b/sdk/objc/api/peerconnection/RTCDevice.h @@ -12,15 +12,24 @@ #import "RTCMacros.h" +/** + * Represents the device type + */ +typedef NS_ENUM(NSInteger, RTCDeviceType) { + RTCDeviceTypeOutput, + RTCDeviceTypeInput, +}; + NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @interface RTCDevice : NSObject -+ (instancetype)defaultDevice; ++ (instancetype)defaultDeviceWithType:(RTCDeviceType)type; - (instancetype)init NS_UNAVAILABLE; @property(nonatomic, readonly) BOOL isDefault; +@property(nonatomic, readonly) RTCDeviceType type; @property(nonatomic, copy, readonly) NSString *guid; @property(nonatomic, copy, readonly) NSString *name; diff --git a/sdk/objc/api/peerconnection/RTCDevice.mm b/sdk/objc/api/peerconnection/RTCDevice.mm index 1594b0e029..dac2a1d37e 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.mm +++ b/sdk/objc/api/peerconnection/RTCDevice.mm @@ -15,17 +15,21 @@ @implementation RTCDevice +@synthesize type = _type; @synthesize guid = _guid; @synthesize name = _name; -+ (instancetype)defaultDevice { - return [[self alloc] initWithGUID: kDefaultDeviceId ++ (instancetype)defaultDeviceWithType:(RTCDeviceType)type { + return [[self alloc] initWithType: type + guid: kDefaultDeviceId name: @""]; } -- (instancetype)initWithGUID:(NSString *)guid +- (instancetype)initWithType:(RTCDeviceType)type + guid:(NSString *)guid name:(NSString* )name { if (self = [super init]) { + _type = type; _guid = guid; _name = name; } From aba22275107351d25375f1dec8bb9d57a8cd3db2 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 13 Mar 2022 05:27:11 +0900 Subject: [PATCH 11/24] bypass --- modules/audio_device/audio_device_impl.cc | 21 +++++--- modules/audio_device/audio_device_impl.h | 7 ++- modules/audio_device/include/audio_device.h | 6 ++- .../RTCPeerConnectionFactory+Native.h | 3 +- .../RTCPeerConnectionFactory.mm | 52 +++++++++---------- .../RTCPeerConnectionFactoryBuilder.mm | 3 +- 6 files changed, 53 insertions(+), 39 deletions(-) diff --git a/modules/audio_device/audio_device_impl.cc b/modules/audio_device/audio_device_impl.cc index 9d540f911c..895b484dec 100644 --- a/modules/audio_device/audio_device_impl.cc +++ b/modules/audio_device/audio_device_impl.cc @@ -72,15 +72,17 @@ namespace webrtc { rtc::scoped_refptr AudioDeviceModule::Create( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory) { + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing) { RTC_DLOG(LS_INFO) << __FUNCTION__; - return AudioDeviceModule::CreateForTest(audio_layer, task_queue_factory); + return AudioDeviceModule::CreateForTest(audio_layer, task_queue_factory, bypass_voice_processing); } // static rtc::scoped_refptr AudioDeviceModule::CreateForTest( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory) { + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing) { RTC_DLOG(LS_INFO) << __FUNCTION__; // The "AudioDeviceModule::kWindowsCoreAudio2" audio layer has its own @@ -93,7 +95,7 @@ rtc::scoped_refptr AudioDeviceModule::CreateForTest( // Create the generic reference counted (platform independent) implementation. auto audio_device = rtc::make_ref_counted( - audio_layer, task_queue_factory); + audio_layer, task_queue_factory, bypass_voice_processing); // Ensure that the current platform is supported. if (audio_device->CheckPlatform() == -1) { @@ -116,8 +118,13 @@ rtc::scoped_refptr AudioDeviceModule::CreateForTest( AudioDeviceModuleImpl::AudioDeviceModuleImpl( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory) - : audio_layer_(audio_layer), audio_device_buffer_(task_queue_factory) { + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing) + : audio_layer_(audio_layer), + #if defined(WEBRTC_IOS) + bypass_voice_processing_(bypass_voice_processing), + #endif + audio_device_buffer_(task_queue_factory) { RTC_DLOG(LS_INFO) << __FUNCTION__; } @@ -280,7 +287,7 @@ int32_t AudioDeviceModuleImpl::CreatePlatformSpecificObjects() { #if defined(WEBRTC_IOS) if (audio_layer == kPlatformDefaultAudio) { audio_device_.reset( - new ios_adm::AudioDeviceIOS(/*bypass_voice_processing=*/false)); + new ios_adm::AudioDeviceIOS(/*bypass_voice_processing=*/bypass_voice_processing_)); RTC_LOG(LS_INFO) << "iPhone Audio APIs will be utilized."; } // END #if defined(WEBRTC_IOS) diff --git a/modules/audio_device/audio_device_impl.h b/modules/audio_device/audio_device_impl.h index 45f73dcd65..76a9f670b6 100644 --- a/modules/audio_device/audio_device_impl.h +++ b/modules/audio_device/audio_device_impl.h @@ -43,7 +43,8 @@ class AudioDeviceModuleImpl : public AudioDeviceModuleForTest { int32_t AttachAudioBuffer(); AudioDeviceModuleImpl(AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory); + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing = false); ~AudioDeviceModuleImpl() override; // Retrieve the currently utilized audio layer @@ -165,7 +166,9 @@ class AudioDeviceModuleImpl : public AudioDeviceModuleForTest { AudioLayer audio_layer_; PlatformType platform_type_ = kPlatformNotSupported; bool initialized_ = false; -#if defined(WEBRTC_ANDROID) +#if defined(WEBRTC_IOS) + bool bypass_voice_processing_; +#elif defined(WEBRTC_ANDROID) // Should be declared first to ensure that it outlives other resources. std::unique_ptr audio_manager_android_; #endif diff --git a/modules/audio_device/include/audio_device.h b/modules/audio_device/include/audio_device.h index f82029eb51..28ace03610 100644 --- a/modules/audio_device/include/audio_device.h +++ b/modules/audio_device/include/audio_device.h @@ -45,12 +45,14 @@ class AudioDeviceModule : public rtc::RefCountInterface { // Creates a default ADM for usage in production code. static rtc::scoped_refptr Create( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory); + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing = false); // Creates an ADM with support for extra test methods. Don't use this factory // in production code. static rtc::scoped_refptr CreateForTest( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory); + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing = false); // Retrieve the currently utilized audio layer virtual int32_t ActiveAudioLayer(AudioLayer* audioLayer) const = 0; diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h index b7f5e27ded..0e67a09794 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h @@ -50,7 +50,8 @@ NS_ASSUME_NONNULL_BEGIN audioDeviceModule: (nullable webrtc::AudioDeviceModule *)audioDeviceModule audioProcessingModule: - (rtc::scoped_refptr)audioProcessingModule; + (rtc::scoped_refptr)audioProcessingModule + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing; - (instancetype) initWithNativeAudioEncoderFactory: diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index dae3752bda..b94a2bcfff 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -49,9 +49,6 @@ #include "sdk/objc/native/api/video_encoder_factory.h" #include "sdk/objc/native/src/objc_video_decoder_factory.h" #include "sdk/objc/native/src/objc_video_encoder_factory.h" -#endif - -#if defined(WEBRTC_IOS) #import "sdk/objc/native/api/audio_device_module.h" #endif @@ -73,14 +70,6 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { @synthesize nativeFactory = _nativeFactory; -- (rtc::scoped_refptr)audioDeviceModule:(BOOL)bypassVoiceProcessing { -#if defined(WEBRTC_IOS) - return webrtc::CreateAudioDeviceModule(bypassVoiceProcessing); -#else - return nullptr; -#endif -} - - (RTCAudioDeviceModule *)audioDeviceModule { return [[RTCAudioDeviceModule alloc] initWithNativeModule: _audioDeviceModule]; } @@ -96,8 +85,9 @@ - (instancetype)init { RTCVideoEncoderFactoryH264) alloc] init]) nativeVideoDecoderFactory:webrtc::ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE( RTCVideoDecoderFactoryH264) alloc] init]) - audioDeviceModule:[self audioDeviceModule:false] - audioProcessingModule:nullptr]; + audioDeviceModule:nullptr + audioProcessingModule:nullptr + bypassVoiceProcessing:NO]; #endif } @@ -119,8 +109,9 @@ - (instancetype)init { nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() nativeVideoEncoderFactory:std::move(native_encoder_factory) nativeVideoDecoderFactory:std::move(native_decoder_factory) - audioDeviceModule:[self audioDeviceModule:false] - audioProcessingModule:nullptr]; + audioDeviceModule:nullptr + audioProcessingModule:nullptr + bypassVoiceProcessing:NO]; #endif } @@ -144,8 +135,9 @@ - (instancetype)init { nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() nativeVideoEncoderFactory:std::move(native_encoder_factory) nativeVideoDecoderFactory:std::move(native_decoder_factory) - audioDeviceModule:[self audioDeviceModule:bypassVoiceProcessing] - audioProcessingModule:nullptr]; + audioDeviceModule:nullptr + audioProcessingModule:nullptr + bypassVoiceProcessing:bypassVoiceProcessing]; #endif } @@ -194,14 +186,16 @@ - (instancetype)initWithNativeAudioEncoderFactory: (std::unique_ptr)videoDecoderFactory audioDeviceModule:(webrtc::AudioDeviceModule *)audioDeviceModule audioProcessingModule: - (rtc::scoped_refptr)audioProcessingModule { + (rtc::scoped_refptr)audioProcessingModule + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing { return [self initWithNativeAudioEncoderFactory:audioEncoderFactory nativeAudioDecoderFactory:audioDecoderFactory nativeVideoEncoderFactory:std::move(videoEncoderFactory) nativeVideoDecoderFactory:std::move(videoDecoderFactory) audioDeviceModule:audioDeviceModule audioProcessingModule:audioProcessingModule - networkControllerFactory:nullptr]; + networkControllerFactory:nullptr + bypassVoiceProcessing:NO]; } - (instancetype)initWithNativeAudioEncoderFactory: (rtc::scoped_refptr)audioEncoderFactory @@ -216,7 +210,8 @@ - (instancetype)initWithNativeAudioEncoderFactory: (rtc::scoped_refptr)audioProcessingModule networkControllerFactory: (std::unique_ptr) - networkControllerFactory { + networkControllerFactory + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing { if (self = [self initNative]) { webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.network_thread = _networkThread.get(); @@ -229,12 +224,16 @@ - (instancetype)initWithNativeAudioEncoderFactory: dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); dependencies.trials = std::make_unique(); cricket::MediaEngineDependencies media_deps; - - rtc::scoped_refptr adm = webrtc::AudioDeviceModule::Create(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, dependencies.task_queue_factory.get()); - adm->Init(); - _audioDeviceModule = std::move(adm); - - media_deps.adm = std::move(adm); + + // always create ADM on worker thread + auto adm = _workerThread->Invoke>(RTC_FROM_HERE, [&dependencies, &bypassVoiceProcessing]() { + return webrtc::AudioDeviceModule::Create(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + dependencies.task_queue_factory.get(), + bypassVoiceProcessing == YES); + }); + + _audioDeviceModule = adm; + media_deps.adm = adm; media_deps.task_queue_factory = dependencies.task_queue_factory.get(); media_deps.audio_encoder_factory = std::move(audioEncoderFactory); media_deps.audio_decoder_factory = std::move(audioDecoderFactory); @@ -252,6 +251,7 @@ - (instancetype)initWithNativeAudioEncoderFactory: std::make_unique(dependencies.task_queue_factory.get()); dependencies.network_controller_factory = std::move(networkControllerFactory); #endif + _nativeFactory = webrtc::CreateModularPeerConnectionFactory(std::move(dependencies)); NSAssert(_nativeFactory, @"Failed to initialize PeerConnectionFactory!"); } diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm index 991ec5a41c..894182568f 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm @@ -39,7 +39,8 @@ + (RTCPeerConnectionFactoryBuilder *)builder { nativeVideoEncoderFactory:std::move(_videoEncoderFactory) nativeVideoDecoderFactory:std::move(_videoDecoderFactory) audioDeviceModule:_audioDeviceModule - audioProcessingModule:_audioProcessingModule]; + audioProcessingModule:_audioProcessingModule + bypassVoiceProcessing:NO]; } - (void)setVideoEncoderFactory:(std::unique_ptr)videoEncoderFactory { From 149c99f0d1da09b5c4a3f6cc48a9db2794e3d2ae Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 13 Mar 2022 21:08:50 +0900 Subject: [PATCH 12/24] refactor --- sdk/BUILD.gn | 13 +++++--- sdk/objc/api/peerconnection/RTCAudioDevice.h | 23 +++++++++++++ sdk/objc/api/peerconnection/RTCAudioDevice.mm | 15 +++++++++ .../api/peerconnection/RTCAudioDeviceModule.h | 10 +++--- .../peerconnection/RTCAudioDeviceModule.mm | 23 +++++++------ ...Device+Private.h => RTCIODevice+Private.h} | 9 +++--- .../{RTCDevice.h => RTCIODevice.h} | 20 +++++------- .../{RTCDevice.mm => RTCIODevice.mm} | 32 +++++++++++-------- .../RTCPeerConnectionFactory+Native.h | 3 +- 9 files changed, 95 insertions(+), 53 deletions(-) create mode 100644 sdk/objc/api/peerconnection/RTCAudioDevice.h create mode 100644 sdk/objc/api/peerconnection/RTCAudioDevice.mm rename sdk/objc/api/peerconnection/{RTCDevice+Private.h => RTCIODevice+Private.h} (74%) rename sdk/objc/api/peerconnection/{RTCDevice.h => RTCIODevice.h} (67%) rename sdk/objc/api/peerconnection/{RTCDevice.mm => RTCIODevice.mm} (57%) diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 395e57484b..7b9f71e4ae 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -930,9 +930,10 @@ if (is_ios || is_mac) { "objc/api/peerconnection/RTCAudioDeviceModule.h", "objc/api/peerconnection/RTCAudioDeviceModule+Private.h", "objc/api/peerconnection/RTCAudioDeviceModule.mm", - "objc/api/peerconnection/RTCDevice.h", - "objc/api/peerconnection/RTCDevice+Private.h", - "objc/api/peerconnection/RTCDevice.mm", + "objc/api/peerconnection/RTCIODevice.h", + "objc/api/peerconnection/RTCIODevice.mm", + "objc/api/peerconnection/RTCAudioDevice.h", + "objc/api/peerconnection/RTCAudioDevice.mm", "objc/api/peerconnection/RTCAudioSource+Private.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioSource.mm", @@ -1325,7 +1326,8 @@ if (is_ios || is_mac) { "objc/helpers/RTCDispatcher.h", "objc/helpers/UIDevice+RTCDevice.h", "objc/api/peerconnection/RTCAudioDeviceModule.h", - "objc/api/peerconnection/RTCDevice.h", + "objc/api/peerconnection/RTCAudioDevice.h", + "objc/api/peerconnection/RTCIODevice.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioTrack.h", "objc/api/peerconnection/RTCConfiguration.h", @@ -1441,7 +1443,8 @@ if (is_ios || is_mac) { sources = [ "objc/api/peerconnection/RTCAudioDeviceModule.h", - "objc/api/peerconnection/RTCDevice.h", + "objc/api/peerconnection/RTCAudioDevice.h", + "objc/api/peerconnection/RTCIODevice.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioTrack.h", "objc/api/peerconnection/RTCCertificate.h", diff --git a/sdk/objc/api/peerconnection/RTCAudioDevice.h b/sdk/objc/api/peerconnection/RTCAudioDevice.h new file mode 100644 index 0000000000..ee14d84d1e --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDevice.h @@ -0,0 +1,23 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" +#import "RTCIODevice.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE(RTCAudioDevice) : RTC_OBJC_TYPE(RTCIODevice) + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioDevice.mm b/sdk/objc/api/peerconnection/RTCAudioDevice.mm new file mode 100644 index 0000000000..99c40c134a --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDevice.mm @@ -0,0 +1,15 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioDevice.h" + +@implementation RTCAudioDevice + +@end diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index f65a165bdc..bb2c7add8c 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -12,7 +12,7 @@ #import #import "RTCMacros.h" -#import "RTCDevice.h" +#import "RTCAudioDevice.h" NS_ASSUME_NONNULL_BEGIN @@ -21,15 +21,15 @@ RTC_OBJC_EXPORT - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer; -@property(nonatomic, readonly) NSArray *playoutDevices; -@property(nonatomic, readonly) NSArray *recordingDevices; +@property(nonatomic, readonly) NSArray *playoutDevices; +@property(nonatomic, readonly) NSArray *recordingDevices; @property(nonatomic, readonly) BOOL playing; @property(nonatomic, readonly) BOOL recording; // Executes low-level API's in sequence to switch the device -- (BOOL)switchPlayoutDevice: (nullable RTCDevice *)device; -- (BOOL)switchRecordingDevice: (nullable RTCDevice *)device; +- (BOOL)switchPlayoutDevice: (nullable RTCAudioDevice *)device; +- (BOOL)switchRecordingDevice: (nullable RTCAudioDevice *)device; // Low-level APIs - (BOOL)setPlayoutDevice:(uint16_t) index; diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 289d866275..5cf971b294 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -11,9 +11,8 @@ #include #import "RTCAudioDeviceModule+Private.h" -#import "RTCDevice+Private.h" +#import "RTCIODevice+Private.h" -#include "rtc_base/ref_counted_object.h" #import "sdk/objc/native/api/audio_device_module.h" @implementation RTCAudioDeviceModule { @@ -31,7 +30,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { // _native->CaptureSampleBuffer(sampleBuffer); } -- (NSArray *)playoutDevices { +- (NSArray *)playoutDevices { char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; @@ -45,7 +44,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { _native->PlayoutDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCDevice *device = [[RTCDevice alloc] initWithType:RTCDeviceTypeOutput guid:strGUID name:strName]; + RTCAudioDevice *device = [[RTCAudioDevice alloc] initWithType:RTCIODeviceTypeOutput deviceId:strGUID name:strName]; [result addObject: device]; } } @@ -53,7 +52,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { return result; } -- (NSArray *)recordingDevices { +- (NSArray *)recordingDevices { char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; @@ -67,7 +66,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { _native->RecordingDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCDevice *device = [[RTCDevice alloc] initWithType:RTCDeviceTypeInput guid:strGUID name:strName]; + RTCAudioDevice *device = [[RTCAudioDevice alloc] initWithType:RTCIODeviceTypeInput deviceId:strGUID name:strName]; [result addObject: device]; } } @@ -75,7 +74,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { return result; } -- (BOOL)switchPlayoutDevice: (nullable RTCDevice *)device { +- (BOOL)switchPlayoutDevice: (nullable RTCAudioDevice *)device { NSUInteger index = 0; NSArray *devices = [self playoutDevices]; @@ -85,8 +84,8 @@ - (BOOL)switchPlayoutDevice: (nullable RTCDevice *)device { } if (device != nil) { - index = [devices indexOfObjectPassingTest:^BOOL(RTCDevice *e, NSUInteger i, BOOL *stop) { - return (*stop = [e.guid isEqualToString:device.guid]); + index = [devices indexOfObjectPassingTest:^BOOL(RTCAudioDevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.deviceId isEqualToString:device.deviceId]); }]; if (index == NSNotFound) { return NO; @@ -105,7 +104,7 @@ - (BOOL)switchPlayoutDevice: (nullable RTCDevice *)device { return NO; } -- (BOOL)switchRecordingDevice: (nullable RTCDevice *)device { +- (BOOL)switchRecordingDevice: (nullable RTCAudioDevice *)device { NSUInteger index = 0; NSArray *devices = [self recordingDevices]; @@ -115,8 +114,8 @@ - (BOOL)switchRecordingDevice: (nullable RTCDevice *)device { } if (device != nil) { - index = [devices indexOfObjectPassingTest:^BOOL(RTCDevice *e, NSUInteger i, BOOL *stop) { - return (*stop = [e.guid isEqualToString:device.guid]); + index = [devices indexOfObjectPassingTest:^BOOL(RTCAudioDevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.deviceId isEqualToString:device.deviceId]); }]; if (index == NSNotFound) { return NO; diff --git a/sdk/objc/api/peerconnection/RTCDevice+Private.h b/sdk/objc/api/peerconnection/RTCIODevice+Private.h similarity index 74% rename from sdk/objc/api/peerconnection/RTCDevice+Private.h rename to sdk/objc/api/peerconnection/RTCIODevice+Private.h index cc41bc0454..640559b0b6 100644 --- a/sdk/objc/api/peerconnection/RTCDevice+Private.h +++ b/sdk/objc/api/peerconnection/RTCIODevice+Private.h @@ -8,14 +8,15 @@ * be found in the AUTHORS file in the root of the source tree. */ -#import "RTCDevice.h" +#import "RTCAudioDevice.h" +#import "RTCIODevice.h" NS_ASSUME_NONNULL_BEGIN -@interface RTCDevice () +@interface RTCIODevice () -- (instancetype)initWithType:(RTCDeviceType)type - guid:(NSString *)guid +- (instancetype)initWithType:(RTCIODeviceType)type + deviceId:(NSString *)deviceId name:(NSString* )name; @end diff --git a/sdk/objc/api/peerconnection/RTCDevice.h b/sdk/objc/api/peerconnection/RTCIODevice.h similarity index 67% rename from sdk/objc/api/peerconnection/RTCDevice.h rename to sdk/objc/api/peerconnection/RTCIODevice.h index 51439441fa..1ca0455c16 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.h +++ b/sdk/objc/api/peerconnection/RTCIODevice.h @@ -12,25 +12,21 @@ #import "RTCMacros.h" -/** - * Represents the device type - */ -typedef NS_ENUM(NSInteger, RTCDeviceType) { - RTCDeviceTypeOutput, - RTCDeviceTypeInput, -}; - NS_ASSUME_NONNULL_BEGIN +typedef NS_ENUM(NSInteger, RTCIODeviceType) { + RTCIODeviceTypeOutput, + RTCIODeviceTypeInput, +}; + RTC_OBJC_EXPORT -@interface RTCDevice : NSObject +@interface RTC_OBJC_TYPE(RTCIODevice) : NSObject -+ (instancetype)defaultDeviceWithType:(RTCDeviceType)type; - (instancetype)init NS_UNAVAILABLE; @property(nonatomic, readonly) BOOL isDefault; -@property(nonatomic, readonly) RTCDeviceType type; -@property(nonatomic, copy, readonly) NSString *guid; +@property(nonatomic, readonly) RTCIODeviceType type; +@property(nonatomic, copy, readonly) NSString *deviceId; @property(nonatomic, copy, readonly) NSString *name; @end diff --git a/sdk/objc/api/peerconnection/RTCDevice.mm b/sdk/objc/api/peerconnection/RTCIODevice.mm similarity index 57% rename from sdk/objc/api/peerconnection/RTCDevice.mm rename to sdk/objc/api/peerconnection/RTCIODevice.mm index dac2a1d37e..da8e463de2 100644 --- a/sdk/objc/api/peerconnection/RTCDevice.mm +++ b/sdk/objc/api/peerconnection/RTCIODevice.mm @@ -8,39 +8,43 @@ * be found in the AUTHORS file in the root of the source tree. */ -#import "RTCDevice+Private.h" -#include "rtc_base/ref_counted_object.h" +#import "RTCIODevice.h" +#import "RTCIODevice+Private.h" NSString *const kDefaultDeviceId = @"default"; -@implementation RTCDevice +@implementation RTCIODevice @synthesize type = _type; -@synthesize guid = _guid; +@synthesize deviceId = _deviceId; @synthesize name = _name; -+ (instancetype)defaultDeviceWithType:(RTCDeviceType)type { ++ (instancetype)defaultDeviceWithType: (RTCIODeviceType)type { return [[self alloc] initWithType: type - guid: kDefaultDeviceId + deviceId: kDefaultDeviceId name: @""]; } -- (instancetype)initWithType:(RTCDeviceType)type - guid:(NSString *)guid - name:(NSString* )name { +- (instancetype)initWithType: (RTCIODeviceType)type + deviceId: (NSString *)deviceId + name: (NSString* )name { if (self = [super init]) { _type = type; - _guid = guid; + _deviceId = deviceId; _name = name; } return self; } +#pragma mark - IODevice + - (BOOL)isDefault { - return [_guid isEqualToString: kDefaultDeviceId]; + return [_deviceId isEqualToString: kDefaultDeviceId]; } -- (BOOL)isEqual:(id)object { +#pragma mark - Equatable + +- (BOOL)isEqual: (id)object { if (self == object) { return YES; } @@ -51,11 +55,11 @@ - (BOOL)isEqual:(id)object { return NO; } - return [_guid isEqualToString:((RTC_OBJC_TYPE(RTCDevice) *)object).guid]; + return [_deviceId isEqualToString:((RTC_OBJC_TYPE(RTCIODevice) *)object).deviceId]; } - (NSUInteger)hash { - return [_guid hash]; + return [_deviceId hash]; } @end diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h index 0e67a09794..902925936b 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h @@ -66,7 +66,8 @@ NS_ASSUME_NONNULL_BEGIN audioProcessingModule: (rtc::scoped_refptr)audioProcessingModule networkControllerFactory:(std::unique_ptr) - networkControllerFactory; + networkControllerFactory + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing; - (instancetype) initWithEncoderFactory:(nullable id)encoderFactory From a00a74cbadbaf6042427ac79997c319928f387fc Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 14 Mar 2022 17:39:58 +0900 Subject: [PATCH 13/24] fix --- sdk/objc/api/peerconnection/RTCIODevice.h | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/objc/api/peerconnection/RTCIODevice.h b/sdk/objc/api/peerconnection/RTCIODevice.h index 1ca0455c16..af4bd207f4 100644 --- a/sdk/objc/api/peerconnection/RTCIODevice.h +++ b/sdk/objc/api/peerconnection/RTCIODevice.h @@ -22,6 +22,7 @@ typedef NS_ENUM(NSInteger, RTCIODeviceType) { RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE(RTCIODevice) : NSObject ++ (instancetype)defaultDeviceWithType: (RTCIODeviceType)type; - (instancetype)init NS_UNAVAILABLE; @property(nonatomic, readonly) BOOL isDefault; From e5410c8811ff41cd6ff556eb29ddf89b98b0ea6b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 14 Mar 2022 18:46:22 +0900 Subject: [PATCH 14/24] append Audio to thread labels --- modules/audio_device/mac/audio_device_mac.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index 64ee58b9df..cae1bca06c 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -1308,7 +1308,7 @@ int32_t AudioDeviceMac::StartRecording() { while (CaptureWorkerThread()) { } }, - "CaptureWorkerThread", + "Audio_CaptureWorkerThread", rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime)); OSStatus err = noErr; @@ -1445,7 +1445,7 @@ int32_t AudioDeviceMac::StartPlayout() { while (RenderWorkerThread()) { } }, - "RenderWorkerThread", + "Audio_RenderWorkerThread", rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime)); if (_twoDevices || !_recording) { From c4f1305dbe9591d6a9f0063c1850b8f6fcf06533 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 15 Mar 2022 00:20:32 +0900 Subject: [PATCH 15/24] ref --- .../api/peerconnection/RTCAudioDeviceModule.h | 20 ++------- .../peerconnection/RTCAudioDeviceModule.mm | 44 +++---------------- 2 files changed, 10 insertions(+), 54 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index bb2c7add8c..8028fbb9a3 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -21,27 +21,15 @@ RTC_OBJC_EXPORT - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer; -@property(nonatomic, readonly) NSArray *playoutDevices; -@property(nonatomic, readonly) NSArray *recordingDevices; +@property(nonatomic, readonly) NSArray *outputDevices; +@property(nonatomic, readonly) NSArray *inputDevices; @property(nonatomic, readonly) BOOL playing; @property(nonatomic, readonly) BOOL recording; // Executes low-level API's in sequence to switch the device -- (BOOL)switchPlayoutDevice: (nullable RTCAudioDevice *)device; -- (BOOL)switchRecordingDevice: (nullable RTCAudioDevice *)device; - -// Low-level APIs -- (BOOL)setPlayoutDevice:(uint16_t) index; -- (BOOL)startPlayout; -- (BOOL)stopPlayout; -- (BOOL)initPlayout; - -// Low-level APIs -- (BOOL)setRecordingDevice:(uint16_t) index; -- (BOOL)startRecording; -- (BOOL)stopRecording; -- (BOOL)initRecording; +- (BOOL)setOutputDevice: (nullable RTCAudioDevice *)device; +- (BOOL)setInputDevice: (nullable RTCAudioDevice *)device; @end diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 5cf971b294..7185757499 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -30,7 +30,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { // _native->CaptureSampleBuffer(sampleBuffer); } -- (NSArray *)playoutDevices { +- (NSArray *)outputDevices { char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; @@ -52,7 +52,7 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { return result; } -- (NSArray *)recordingDevices { +- (NSArray *)inputDevices { char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; @@ -74,10 +74,10 @@ - (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { return result; } -- (BOOL)switchPlayoutDevice: (nullable RTCAudioDevice *)device { +- (BOOL)setOutputDevice: (nullable RTCAudioDevice *)device { NSUInteger index = 0; - NSArray *devices = [self playoutDevices]; + NSArray *devices = [self outputDevices]; if ([devices count] == 0) { return NO; @@ -104,10 +104,10 @@ - (BOOL)switchPlayoutDevice: (nullable RTCAudioDevice *)device { return NO; } -- (BOOL)switchRecordingDevice: (nullable RTCAudioDevice *)device { +- (BOOL)setInputDevice: (nullable RTCAudioDevice *)device { NSUInteger index = 0; - NSArray *devices = [self recordingDevices]; + NSArray *devices = [self inputDevices]; if ([devices count] == 0) { return NO; @@ -138,40 +138,8 @@ - (BOOL)playing { return _native->Playing(); } -- (BOOL)setPlayoutDevice:(uint16_t) index { - return _native->SetPlayoutDevice(index) == 0; -} - -- (BOOL)startPlayout { - return _native->StartPlayout() == 0; -} - -- (BOOL)stopPlayout { - return _native->StopPlayout() == 0; -} - -- (BOOL)initPlayout { - return _native->InitPlayout() == 0; -} - - (BOOL)recording { return _native->Recording(); } -- (BOOL)setRecordingDevice:(uint16_t) index { - return _native->SetRecordingDevice(index) == 0; -} - -- (BOOL)startRecording { - return _native->StartRecording() == 0; -} - -- (BOOL)stopRecording { - return _native->StopRecording() == 0; -} - -- (BOOL)initRecording { - return _native->InitRecording() == 0; -} - @end From a35ce5667688087508154489a930f2a3794a4af7 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 15 Mar 2022 04:17:11 +0900 Subject: [PATCH 16/24] lk headers --- sdk/objc/api/peerconnection/RTCAudioDevice.h | 18 ++++++++++++------ sdk/objc/api/peerconnection/RTCAudioDevice.mm | 18 ++++++++++++------ .../RTCAudioDeviceModule+Private.h | 18 ++++++++++++------ .../api/peerconnection/RTCAudioDeviceModule.h | 18 ++++++++++++------ .../api/peerconnection/RTCAudioDeviceModule.mm | 18 ++++++++++++------ .../api/peerconnection/RTCIODevice+Private.h | 18 ++++++++++++------ sdk/objc/api/peerconnection/RTCIODevice.h | 18 ++++++++++++------ sdk/objc/api/peerconnection/RTCIODevice.mm | 18 ++++++++++++------ 8 files changed, 96 insertions(+), 48 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioDevice.h b/sdk/objc/api/peerconnection/RTCAudioDevice.h index ee14d84d1e..4191beecb7 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDevice.h +++ b/sdk/objc/api/peerconnection/RTCAudioDevice.h @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import diff --git a/sdk/objc/api/peerconnection/RTCAudioDevice.mm b/sdk/objc/api/peerconnection/RTCAudioDevice.mm index 99c40c134a..f30536cf30 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDevice.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDevice.mm @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import "RTCAudioDevice.h" diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h index f4a8ff289c..cb5bb3974b 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import "RTCAudioDeviceModule.h" diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index 8028fbb9a3..6c6cc8fde7 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 7185757499..2a159443ae 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include diff --git a/sdk/objc/api/peerconnection/RTCIODevice+Private.h b/sdk/objc/api/peerconnection/RTCIODevice+Private.h index 640559b0b6..e6dbb2f968 100644 --- a/sdk/objc/api/peerconnection/RTCIODevice+Private.h +++ b/sdk/objc/api/peerconnection/RTCIODevice+Private.h @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import "RTCAudioDevice.h" diff --git a/sdk/objc/api/peerconnection/RTCIODevice.h b/sdk/objc/api/peerconnection/RTCIODevice.h index af4bd207f4..f44d532081 100644 --- a/sdk/objc/api/peerconnection/RTCIODevice.h +++ b/sdk/objc/api/peerconnection/RTCIODevice.h @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import diff --git a/sdk/objc/api/peerconnection/RTCIODevice.mm b/sdk/objc/api/peerconnection/RTCIODevice.mm index da8e463de2..27e1255e8e 100644 --- a/sdk/objc/api/peerconnection/RTCIODevice.mm +++ b/sdk/objc/api/peerconnection/RTCIODevice.mm @@ -1,11 +1,17 @@ /* - * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * Copyright 2022 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import "RTCIODevice.h" From bdf5e7ef522f864edc2b5573384a2125945e3b96 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 18 Mar 2022 23:21:36 +0900 Subject: [PATCH 17/24] low level apis --- .../api/peerconnection/RTCAudioDeviceModule.h | 7 +++++ .../peerconnection/RTCAudioDeviceModule.mm | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index 6c6cc8fde7..09ca50676c 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -37,6 +37,13 @@ RTC_OBJC_EXPORT - (BOOL)setOutputDevice: (nullable RTCAudioDevice *)device; - (BOOL)setInputDevice: (nullable RTCAudioDevice *)device; +- (BOOL)startPlayout; +- (BOOL)stopPlayout; +- (BOOL)initPlayout; +- (BOOL)startRecording; +- (BOOL)stopRecording; +- (BOOL)initRecording; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 2a159443ae..fe8cc5fd2c 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -148,4 +148,30 @@ - (BOOL)recording { return _native->Recording(); } +#pragma mark - + +- (BOOL)startPlayout { + return _native->StartPlayout() == 0; +} + +- (BOOL)stopPlayout { + return _native->StopPlayout() == 0; +} + +- (BOOL)initPlayout { + return _native->InitPlayout() == 0; +} + +- (BOOL)startRecording { + return _native->StartRecording() == 0; +} + +- (BOOL)stopRecording { + return _native->StopRecording() == 0; +} + +- (BOOL)initRecording { + return _native->InitRecording() == 0; +} + @end From bda204cbef5dbbeb659c1412bfa1dea2ec2524ce Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 27 May 2022 19:20:34 +0900 Subject: [PATCH 18/24] fix thread checks Some methods of ADM needs to be run on worker thread, otherwise RTC's thread check will fail. --- .../RTCAudioDeviceModule+Private.h | 5 +- .../api/peerconnection/RTCAudioDeviceModule.h | 2 - .../peerconnection/RTCAudioDeviceModule.mm | 221 +++++++++++------- .../peerconnection/RTCPeerConnectionFactory.h | 3 +- .../RTCPeerConnectionFactory.mm | 3 +- 5 files changed, 141 insertions(+), 93 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h index cb5bb3974b..4eb91b93c7 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h @@ -17,11 +17,14 @@ #import "RTCAudioDeviceModule.h" #import "sdk/objc/native/api/audio_device_module.h" +#include "rtc_base/thread.h" + NS_ASSUME_NONNULL_BEGIN @interface RTCAudioDeviceModule () -- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module; +- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module + workerThread:(rtc::Thread *)workerThread; @end diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index 09ca50676c..fe7d91ad10 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -25,8 +25,6 @@ NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCAudioDeviceModule) : NSObject -- (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer; - @property(nonatomic, readonly) NSArray *outputDevices; @property(nonatomic, readonly) NSArray *inputDevices; diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index fe8cc5fd2c..1f0e49a013 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -16,162 +16,207 @@ #include +#import "RTCAudioDeviceModule.h" #import "RTCAudioDeviceModule+Private.h" #import "RTCIODevice+Private.h" #import "sdk/objc/native/api/audio_device_module.h" -@implementation RTCAudioDeviceModule { +@implementation RTC_OBJC_TYPE (RTCAudioDeviceModule) { + rtc::Thread *_workerThread; rtc::scoped_refptr _native; } -- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module { +- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module + workerThread:(rtc::Thread * )workerThread { self = [super init]; _native = module; + _workerThread = workerThread; return self; } -- (void)captureSampleBuffer:(CMSampleBufferRef)sampleBuffer { - // TODO: Implement - // _native->CaptureSampleBuffer(sampleBuffer); -} - - (NSArray *)outputDevices { - - char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; - char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; - - NSMutableArray *result = [NSMutableArray array]; - int16_t count = _native->PlayoutDevices(); - - if (count > 0) { - for (int i = 0; i < count; i++) { - _native->PlayoutDeviceName(i, name, guid); - NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; - NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCAudioDevice *device = [[RTCAudioDevice alloc] initWithType:RTCIODeviceTypeOutput deviceId:strGUID name:strName]; - [result addObject: device]; - } - } - - return result; + return _workerThread->Invoke *>(RTC_FROM_HERE, [self] { + return [self _outputDevices]; + }); } - (NSArray *)inputDevices { - - char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; - char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; - - NSMutableArray *result = [NSMutableArray array]; - - int16_t count = _native->RecordingDevices(); - if (count > 0) { - for (int i = 0; i < count; i++) { - _native->RecordingDeviceName(i, name, guid); - NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; - NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCAudioDevice *device = [[RTCAudioDevice alloc] initWithType:RTCIODeviceTypeInput deviceId:strGUID name:strName]; - [result addObject: device]; - } - } - - return result; + return _workerThread->Invoke *>(RTC_FROM_HERE, [self] { + return [self _inputDevices]; + }); } - (BOOL)setOutputDevice: (nullable RTCAudioDevice *)device { - NSUInteger index = 0; - NSArray *devices = [self outputDevices]; + return _workerThread->Invoke(RTC_FROM_HERE, [self, device] { - if ([devices count] == 0) { - return NO; - } + NSUInteger index = 0; + NSArray *devices = [self _outputDevices]; - if (device != nil) { - index = [devices indexOfObjectPassingTest:^BOOL(RTCAudioDevice *e, NSUInteger i, BOOL *stop) { - return (*stop = [e.deviceId isEqualToString:device.deviceId]); - }]; - if (index == NSNotFound) { + if ([devices count] == 0) { return NO; } - } - _native->StopPlayout(); + if (device != nil) { + index = [devices indexOfObjectPassingTest:^BOOL(RTCAudioDevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.deviceId isEqualToString:device.deviceId]); + }]; + if (index == NSNotFound) { + return NO; + } + } - if (_native->SetPlayoutDevice(index) == 0 - && _native->InitPlayout() == 0 - && _native->StartPlayout() == 0) { + _native->StopPlayout(); - return YES; - } + if (_native->SetPlayoutDevice(index) == 0 + && _native->InitPlayout() == 0 + && _native->StartPlayout() == 0) { - return NO; + return YES; + } + + return NO; + }); } - (BOOL)setInputDevice: (nullable RTCAudioDevice *)device { - NSUInteger index = 0; - NSArray *devices = [self inputDevices]; + return _workerThread->Invoke(RTC_FROM_HERE, [self, device] { - if ([devices count] == 0) { - return NO; - } + NSUInteger index = 0; + NSArray *devices = [self _inputDevices]; - if (device != nil) { - index = [devices indexOfObjectPassingTest:^BOOL(RTCAudioDevice *e, NSUInteger i, BOOL *stop) { - return (*stop = [e.deviceId isEqualToString:device.deviceId]); - }]; - if (index == NSNotFound) { + if ([devices count] == 0) { return NO; } - } - _native->StopRecording(); + if (device != nil) { + index = [devices indexOfObjectPassingTest:^BOOL(RTCAudioDevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.deviceId isEqualToString:device.deviceId]); + }]; + if (index == NSNotFound) { + return NO; + } + } + + _native->StopRecording(); - if (_native->SetRecordingDevice(index) == 0 - && _native->InitRecording() == 0 - && _native->StartRecording() == 0) { + if (_native->SetRecordingDevice(index) == 0 + && _native->InitRecording() == 0 + && _native->StartRecording() == 0) { - return YES; - } + return YES; + } - return NO; + return NO; + }); } - (BOOL)playing { - return _native->Playing(); + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->Playing(); + }); } - (BOOL)recording { - return _native->Recording(); + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->Recording(); + }); } -#pragma mark - +#pragma mark - Low-level access - (BOOL)startPlayout { - return _native->StartPlayout() == 0; + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->StartPlayout() == 0; + }); } - (BOOL)stopPlayout { - return _native->StopPlayout() == 0; + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->StopPlayout() == 0; + }); } - (BOOL)initPlayout { - return _native->InitPlayout() == 0; + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->InitPlayout() == 0; + }); } - (BOOL)startRecording { - return _native->StartRecording() == 0; + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->StartRecording() == 0; + }); } - (BOOL)stopRecording { - return _native->StopRecording() == 0; + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->StopRecording() == 0; + }); } - (BOOL)initRecording { - return _native->InitRecording() == 0; + + return _workerThread->Invoke(RTC_FROM_HERE, [self] { + return _native->InitRecording() == 0; + }); +} + +#pragma mark - Private + +- (NSArray *)_outputDevices { + + char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; + char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; + + NSMutableArray *result = [NSMutableArray array]; + + int16_t count = _native->PlayoutDevices(); + + if (count > 0) { + for (int i = 0; i < count; i++) { + _native->PlayoutDeviceName(i, name, guid); + NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; + NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; + RTCAudioDevice *device = [[RTCAudioDevice alloc] initWithType:RTCIODeviceTypeOutput deviceId:strGUID name:strName]; + [result addObject: device]; + } + } + + return result; +} + +- (NSArray *)_inputDevices { + + char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; + char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; + + NSMutableArray *result = [NSMutableArray array]; + + int16_t count = _native->RecordingDevices(); + + if (count > 0) { + for (int i = 0; i < count; i++) { + _native->RecordingDeviceName(i, name, guid); + NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; + NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; + RTCAudioDevice *device = [[RTCAudioDevice alloc] initWithType:RTCIODeviceTypeInput deviceId:strGUID name:strName]; + [result addObject: device]; + } + } + + return result; } @end diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h index b92e62ea90..d3fb5b7ca2 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h @@ -11,7 +11,6 @@ #import #import "RTCMacros.h" -#import "RTCAudioDeviceModule.h" NS_ASSUME_NONNULL_BEGIN @@ -24,6 +23,8 @@ NS_ASSUME_NONNULL_BEGIN @class RTC_OBJC_TYPE(RTCVideoSource); @class RTC_OBJC_TYPE(RTCVideoTrack); @class RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions); +@class RTC_OBJC_TYPE(RTCAudioDeviceModule); + @protocol RTC_OBJC_TYPE (RTCPeerConnectionDelegate); @protocol RTC_OBJC_TYPE diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index b94a2bcfff..15935cf549 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -71,7 +71,8 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { @synthesize nativeFactory = _nativeFactory; - (RTCAudioDeviceModule *)audioDeviceModule { - return [[RTCAudioDeviceModule alloc] initWithNativeModule: _audioDeviceModule]; + return [[RTCAudioDeviceModule alloc] initWithNativeModule: _audioDeviceModule + workerThread: _workerThread.get()]; } - (instancetype)init { From e25aa389cb8db424a766067454a06eb485b312b3 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 31 May 2022 13:02:13 +0900 Subject: [PATCH 19/24] switch to default device when removed --- modules/audio_device/mac/audio_device_mac.cc | 40 +++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index a07f279956..8a5a6a7fe6 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -1892,9 +1892,23 @@ int32_t AudioDeviceMac::HandleDeviceChange() { &size, &deviceIsAlive); if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { - RTC_LOG(LS_WARNING) << "Capture device is not alive (probably removed)"; - AtomicSet32(&_captureDeviceIsAlive, 0); - _mixerManager.CloseMicrophone(); + RTC_LOG(LS_WARNING) << "Audio input device is not alive (probably removed) deviceID: " << _inputDeviceID; + //AtomicSet32(&_captureDeviceIsAlive, 0); + //_mixerManager.CloseMicrophone(); + + // Logic to switch to default device (if exists) + // when the current device is not alive anymore + int32_t captureDeviceIsAlive = AtomicGet32(&_captureDeviceIsAlive); + bool wasRecording = _recording && captureDeviceIsAlive == 1; + + StopRecording(); + + // was playing & default device exists + if (wasRecording && SetRecordingDevice(0) == 0) { + InitRecording(); + StartRecording(); + } + } else if (err != noErr) { logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", (const char*)&err); @@ -1911,9 +1925,23 @@ int32_t AudioDeviceMac::HandleDeviceChange() { &size, &deviceIsAlive); if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { - RTC_LOG(LS_WARNING) << "Render device is not alive (probably removed)"; - AtomicSet32(&_renderDeviceIsAlive, 0); - _mixerManager.CloseSpeaker(); + RTC_LOG(LS_WARNING) << "Audio output device is not alive (probably removed) deviceID: " << _outputDeviceID; + // AtomicSet32(&_renderDeviceIsAlive, 0); + // _mixerManager.CloseSpeaker(); + + // Logic to switch to default device (if exists) + // when the current device is not alive anymore + int32_t renderDeviceIsAlive = AtomicGet32(&_renderDeviceIsAlive); + bool wasPlaying = _playing && renderDeviceIsAlive == 1; + + StopPlayout(); + + // was playing & default device exists + if (wasPlaying && SetPlayoutDevice(0) == 0) { + InitPlayout(); + StartPlayout(); + } + } else if (err != noErr) { logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", (const char*)&err); From d30b2e5cbb4ab8638408584103efcfa0f119d974 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 31 May 2022 13:23:33 +0900 Subject: [PATCH 20/24] close mixerManager if didn't switch to default device --- modules/audio_device/mac/audio_device_mac.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index 8a5a6a7fe6..7f8ed1293b 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -1894,7 +1894,6 @@ int32_t AudioDeviceMac::HandleDeviceChange() { if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { RTC_LOG(LS_WARNING) << "Audio input device is not alive (probably removed) deviceID: " << _inputDeviceID; //AtomicSet32(&_captureDeviceIsAlive, 0); - //_mixerManager.CloseMicrophone(); // Logic to switch to default device (if exists) // when the current device is not alive anymore @@ -1907,6 +1906,8 @@ int32_t AudioDeviceMac::HandleDeviceChange() { if (wasRecording && SetRecordingDevice(0) == 0) { InitRecording(); StartRecording(); + } else { + _mixerManager.CloseMicrophone(); } } else if (err != noErr) { @@ -1926,8 +1927,7 @@ int32_t AudioDeviceMac::HandleDeviceChange() { if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { RTC_LOG(LS_WARNING) << "Audio output device is not alive (probably removed) deviceID: " << _outputDeviceID; - // AtomicSet32(&_renderDeviceIsAlive, 0); - // _mixerManager.CloseSpeaker(); + // AtomicSet32(&_renderDeviceIsAlive, 0); // StopPlayout() does this // Logic to switch to default device (if exists) // when the current device is not alive anymore @@ -1940,6 +1940,8 @@ int32_t AudioDeviceMac::HandleDeviceChange() { if (wasPlaying && SetPlayoutDevice(0) == 0) { InitPlayout(); StartPlayout(); + } else { + _mixerManager.CloseSpeaker(); } } else if (err != noErr) { From 1e0ef352713fd98ee44b13c10dd8225df07093e6 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 1 Jun 2022 02:02:13 +0900 Subject: [PATCH 21/24] default audio device switched --- modules/audio_device/mac/audio_device_mac.cc | 102 +++++++++++++++++-- modules/audio_device/mac/audio_device_mac.h | 2 + 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index 7f8ed1293b..1f5dd9e87c 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -294,8 +294,11 @@ AudioDeviceGeneric::InitStatus AudioDeviceMac::Init() { // but now must be explicitly specified. HAL would otherwise try to use the // main thread to issue notifications. AudioObjectPropertyAddress propertyAddress = { - kAudioHardwarePropertyRunLoop, kAudioObjectPropertyScopeGlobal, - kAudioObjectPropertyElementMaster}; + kAudioHardwarePropertyRunLoop, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster + }; + CFRunLoopRef runLoop = NULL; UInt32 size = sizeof(CFRunLoopRef); int aoerr = AudioObjectSetPropertyData( @@ -311,6 +314,16 @@ AudioDeviceGeneric::InitStatus AudioDeviceMac::Init() { WEBRTC_CA_LOG_ERR(AudioObjectAddPropertyListener( kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + // Listen for default output device change. + propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; + WEBRTC_CA_LOG_ERR(AudioObjectAddPropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + + // Listen for default input device change. + propertyAddress.mSelector = kAudioHardwarePropertyDefaultInputDevice; + WEBRTC_CA_LOG_ERR(AudioObjectAddPropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + _initialized = true; return InitStatus::OK; @@ -337,9 +350,21 @@ int32_t AudioDeviceMac::Terminate() { OSStatus err = noErr; int retVal = 0; + // Remove listeners for global scope. AudioObjectPropertyAddress propertyAddress = { - kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, - kAudioObjectPropertyElementMaster}; + kAudioHardwarePropertyDevices, // selector + kAudioObjectPropertyScopeGlobal, // scope + kAudioObjectPropertyElementMaster // element + }; + + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + + propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + + propertyAddress.mSelector = kAudioHardwarePropertyDefaultInputDevice; WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); @@ -1356,7 +1381,11 @@ int32_t AudioDeviceMac::StopRecording() { // Remove listeners. AudioObjectPropertyAddress propertyAddress = { - kAudioDevicePropertyStreamFormat, kAudioDevicePropertyScopeInput, 0}; + kAudioDevicePropertyStreamFormat, // selector + kAudioDevicePropertyScopeInput, // scope + 0, // element + }; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( _inputDeviceID, &propertyAddress, &objectListenerProc, this)); @@ -1469,7 +1498,11 @@ int32_t AudioDeviceMac::StopPlayout() { // Remove listeners. AudioObjectPropertyAddress propertyAddress = { - kAudioDevicePropertyStreamFormat, kAudioDevicePropertyScopeOutput, 0}; + kAudioDevicePropertyStreamFormat, // selector + kAudioDevicePropertyScopeOutput, // scope + 0, // element + }; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( _outputDeviceID, &propertyAddress, &objectListenerProc, this)); @@ -1504,8 +1537,11 @@ int32_t AudioDeviceMac::GetNumberDevices(const AudioObjectPropertyScope scope, OSStatus err = noErr; AudioObjectPropertyAddress propertyAddress = { - kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, - kAudioObjectPropertyElementMaster}; + kAudioHardwarePropertyDevices, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster, + }; + UInt32 size = 0; WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyDataSize( kAudioObjectSystemObject, &propertyAddress, 0, NULL, &size)); @@ -1870,6 +1906,56 @@ OSStatus AudioDeviceMac::implObjectListenerProc( HandleDataSourceChange(objectId, addresses[i]); } else if (addresses[i].mSelector == kAudioDeviceProcessorOverload) { HandleProcessorOverload(addresses[i]); + } else if (addresses[i].mSelector == kAudioHardwarePropertyDefaultOutputDevice) { + RTC_LOG(LS_VERBOSE) << "kAudioHardwarePropertyDefaultOutputDevice"; + // default audio output device changed + HandleDefaultOutputDeviceChange(); + } else if (addresses[i].mSelector == kAudioHardwarePropertyDefaultInputDevice) { + RTC_LOG(LS_VERBOSE) << "kAudioHardwarePropertyDefaultInputDevice"; + // default audio input device changed + HandleDefaultInputDeviceChange(); + } + } + + return 0; +} + +int32_t AudioDeviceMac::HandleDefaultOutputDeviceChange() { + + if (SpeakerIsInitialized()) { + RTC_LOG(LS_WARNING) << "Default audio output device has changed"; + int32_t renderDeviceIsAlive = AtomicGet32(&_renderDeviceIsAlive); + bool wasPlaying = _playing && renderDeviceIsAlive == 1; + + if (wasPlaying && _outputDeviceIsSpecified && _outputDeviceIndex == 0) { + + StopPlayout(); + + // default is already selected _outputDeviceIndex(0) + // re-init and start playout + InitPlayout(); + StartPlayout(); + } + } + + return 0; +} + +int32_t AudioDeviceMac::HandleDefaultInputDeviceChange() { + + if (MicrophoneIsInitialized()) { + RTC_LOG(LS_WARNING) << "Default audio input device has changed"; + int32_t captureDeviceIsAlive = AtomicGet32(&_captureDeviceIsAlive); + bool wasRecording = _recording && captureDeviceIsAlive == 1; + + if (wasRecording && _inputDeviceIsSpecified && _inputDeviceIndex == 0) { + + StopRecording(); + + // default is already selected _inputDeviceIndex(0) + // re-init and start recording + InitRecording(); + StartRecording(); } } diff --git a/modules/audio_device/mac/audio_device_mac.h b/modules/audio_device/mac/audio_device_mac.h index 241714b851..8d97d72b64 100644 --- a/modules/audio_device/mac/audio_device_mac.h +++ b/modules/audio_device/mac/audio_device_mac.h @@ -200,6 +200,8 @@ class AudioDeviceMac : public AudioDeviceGeneric { const AudioObjectPropertyAddress addresses[]); int32_t HandleDeviceChange(); + int32_t HandleDefaultOutputDeviceChange(); + int32_t HandleDefaultInputDeviceChange(); int32_t HandleStreamFormatChange(AudioObjectID objectId, AudioObjectPropertyAddress propertyAddress); From 6a99ab34beb351d7fe20ce16ad074886af01f5c5 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 7 Jun 2022 17:37:27 +0900 Subject: [PATCH 22/24] expose devices update handler --- .../audio_device_data_observer.cc | 4 +++ modules/audio_device/audio_device_generic.cc | 5 ++++ modules/audio_device/audio_device_generic.h | 5 ++++ modules/audio_device/audio_device_impl.cc | 8 ++++++ modules/audio_device/audio_device_impl.h | 2 ++ modules/audio_device/include/audio_device.h | 11 ++++++++ .../audio_device/include/test_audio_device.cc | 5 ++++ modules/audio_device/mac/audio_device_mac.cc | 20 +++++++++++++++ .../api/peerconnection/RTCAudioDeviceModule.h | 4 +++ .../peerconnection/RTCAudioDeviceModule.mm | 25 +++++++++++++++++++ 10 files changed, 89 insertions(+) diff --git a/modules/audio_device/audio_device_data_observer.cc b/modules/audio_device/audio_device_data_observer.cc index be78fd16d7..cd6c070f42 100644 --- a/modules/audio_device/audio_device_data_observer.cc +++ b/modules/audio_device/audio_device_data_observer.cc @@ -288,6 +288,10 @@ class ADMWrapper : public AudioDeviceModule, public AudioTransport { } #endif // WEBRTC_IOS + int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const override { + return impl_->SetAudioDeviceSink(sink); + } + protected: rtc::scoped_refptr impl_; AudioDeviceDataObserver* legacy_observer_ = nullptr; diff --git a/modules/audio_device/audio_device_generic.cc b/modules/audio_device/audio_device_generic.cc index 7b8cfd1734..42eae8defc 100644 --- a/modules/audio_device/audio_device_generic.cc +++ b/modules/audio_device/audio_device_generic.cc @@ -63,4 +63,9 @@ int AudioDeviceGeneric::GetRecordAudioParameters( } #endif // WEBRTC_IOS +int32_t AudioDeviceGeneric::SetAudioDeviceSink(AudioDeviceSink* sink) { + audio_device_module_sink_ = sink; + return 0; +} + } // namespace webrtc diff --git a/modules/audio_device/audio_device_generic.h b/modules/audio_device/audio_device_generic.h index 41e24eb3b0..635f20f5e0 100644 --- a/modules/audio_device/audio_device_generic.h +++ b/modules/audio_device/audio_device_generic.h @@ -135,9 +135,14 @@ class AudioDeviceGeneric { virtual int GetRecordAudioParameters(AudioParameters* params) const; #endif // WEBRTC_IOS + int32_t SetAudioDeviceSink(AudioDeviceSink* sink); + virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) = 0; virtual ~AudioDeviceGeneric() {} + + protected: + AudioDeviceSink* audio_device_module_sink_ = nullptr; }; } // namespace webrtc diff --git a/modules/audio_device/audio_device_impl.cc b/modules/audio_device/audio_device_impl.cc index 895b484dec..6f5fd10571 100644 --- a/modules/audio_device/audio_device_impl.cc +++ b/modules/audio_device/audio_device_impl.cc @@ -944,6 +944,14 @@ int AudioDeviceModuleImpl::GetRecordAudioParameters( } #endif // WEBRTC_IOS +int32_t AudioDeviceModuleImpl::SetAudioDeviceSink(AudioDeviceSink* sink) const { + RTC_LOG(LS_INFO) << __FUNCTION__ << "(" << sink << ")"; + CHECKinitialized_(); + int32_t ok = audio_device_->SetAudioDeviceSink(sink); + RTC_LOG(LS_INFO) << "output: " << ok; + return ok; +} + AudioDeviceModuleImpl::PlatformType AudioDeviceModuleImpl::Platform() const { RTC_LOG(LS_INFO) << __FUNCTION__; return platform_type_; diff --git a/modules/audio_device/audio_device_impl.h b/modules/audio_device/audio_device_impl.h index 76a9f670b6..226f2503c3 100644 --- a/modules/audio_device/audio_device_impl.h +++ b/modules/audio_device/audio_device_impl.h @@ -146,6 +146,8 @@ class AudioDeviceModuleImpl : public AudioDeviceModuleForTest { int GetRecordAudioParameters(AudioParameters* params) const override; #endif // WEBRTC_IOS + int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const override; + #if defined(WEBRTC_ANDROID) // Only use this acccessor for test purposes on Android. AudioManager* GetAndroidAudioManagerForTest() { diff --git a/modules/audio_device/include/audio_device.h b/modules/audio_device/include/audio_device.h index 28ace03610..b3a991aa06 100644 --- a/modules/audio_device/include/audio_device.h +++ b/modules/audio_device/include/audio_device.h @@ -20,6 +20,15 @@ namespace webrtc { class AudioDeviceModuleForTest; +// Sink for callbacks related to a audio device. +class AudioDeviceSink { + public: + virtual ~AudioDeviceSink() = default; + + // input/output devices updated or default device changed + virtual void OnDevicesUpdated() {} +}; + class AudioDeviceModule : public rtc::RefCountInterface { public: enum AudioLayer { @@ -158,6 +167,8 @@ class AudioDeviceModule : public rtc::RefCountInterface { virtual int GetRecordAudioParameters(AudioParameters* params) const = 0; #endif // WEBRTC_IOS + virtual int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const = 0; + protected: ~AudioDeviceModule() override {} }; diff --git a/modules/audio_device/include/test_audio_device.cc b/modules/audio_device/include/test_audio_device.cc index d8ab22f29d..37a4cb0e44 100644 --- a/modules/audio_device/include/test_audio_device.cc +++ b/modules/audio_device/include/test_audio_device.cc @@ -141,6 +141,11 @@ class TestAudioDeviceModuleImpl return capturing_; } + int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const override { + // no-op + return 0; + } + // Blocks until the Renderer refuses to receive data. // Returns false if `timeout_ms` passes before that happens. bool WaitForPlayoutEnd(int timeout_ms = rtc::Event::kForever) override { diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index 1f5dd9e87c..142a246329 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -1936,6 +1936,11 @@ int32_t AudioDeviceMac::HandleDefaultOutputDeviceChange() { InitPlayout(); StartPlayout(); } + + // Notify default output device updated + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); + } } return 0; @@ -1957,6 +1962,11 @@ int32_t AudioDeviceMac::HandleDefaultInputDeviceChange() { InitRecording(); StartRecording(); } + + // Notify default input device updated + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); + } } return 0; @@ -1996,6 +2006,11 @@ int32_t AudioDeviceMac::HandleDeviceChange() { _mixerManager.CloseMicrophone(); } + // Notify input device removed + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); + } + } else if (err != noErr) { logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", (const char*)&err); @@ -2030,6 +2045,11 @@ int32_t AudioDeviceMac::HandleDeviceChange() { _mixerManager.CloseSpeaker(); } + // Notify output device removed + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); + } + } else if (err != noErr) { logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", (const char*)&err); diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index fe7d91ad10..4333167fe0 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -22,6 +22,8 @@ NS_ASSUME_NONNULL_BEGIN +typedef void (^RTCOnAudioDevicesDidUpdate)(); + RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCAudioDeviceModule) : NSObject @@ -35,6 +37,8 @@ RTC_OBJC_EXPORT - (BOOL)setOutputDevice: (nullable RTCAudioDevice *)device; - (BOOL)setInputDevice: (nullable RTCAudioDevice *)device; +- (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler; + - (BOOL)startPlayout; - (BOOL)stopPlayout; - (BOOL)initPlayout; diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 1f0e49a013..8eebb20dca 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -22,9 +22,24 @@ #import "sdk/objc/native/api/audio_device_module.h" +class AudioDeviceSink : public webrtc::AudioDeviceSink { + public: + AudioDeviceSink() {} + + void OnDevicesUpdated() override { + if (callback_handler_) { + callback_handler_(); + } + } + + // private: + RTCOnAudioDevicesDidUpdate callback_handler_; +}; + @implementation RTC_OBJC_TYPE (RTCAudioDeviceModule) { rtc::Thread *_workerThread; rtc::scoped_refptr _native; + AudioDeviceSink *_sink; } - (instancetype)initWithNativeModule:(rtc::scoped_refptr )module @@ -32,6 +47,10 @@ - (instancetype)initWithNativeModule:(rtc::scoped_refptrSetAudioDeviceSink(_sink); + return self; } @@ -173,6 +192,12 @@ - (BOOL)initRecording { }); } +- (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler { + + _sink->callback_handler_ = handler; + return YES; +} + #pragma mark - Private - (NSArray *)_outputDevices { From ccf95cf717b96bfc958fb17839c0070b842358b2 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jun 2022 19:04:13 +0900 Subject: [PATCH 23/24] fix ios compile --- sdk/objc/native/src/audio/audio_device_module_ios.h | 3 +++ sdk/objc/native/src/audio/audio_device_module_ios.mm | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/sdk/objc/native/src/audio/audio_device_module_ios.h b/sdk/objc/native/src/audio/audio_device_module_ios.h index 9bcf114e32..bf1f713df0 100644 --- a/sdk/objc/native/src/audio/audio_device_module_ios.h +++ b/sdk/objc/native/src/audio/audio_device_module_ios.h @@ -130,6 +130,9 @@ class AudioDeviceModuleIOS : public AudioDeviceModule { int GetPlayoutAudioParameters(AudioParameters* params) const override; int GetRecordAudioParameters(AudioParameters* params) const override; #endif // WEBRTC_IOS + + int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const override; + private: const bool bypass_voice_processing_; bool initialized_ = false; diff --git a/sdk/objc/native/src/audio/audio_device_module_ios.mm b/sdk/objc/native/src/audio/audio_device_module_ios.mm index db4ed21f9a..a7b1f680e8 100644 --- a/sdk/objc/native/src/audio/audio_device_module_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_module_ios.mm @@ -667,5 +667,10 @@ return r; } #endif // WEBRTC_IOS + + int32_t AudioDeviceModuleIOS::SetAudioDeviceSink(AudioDeviceSink* sink) const { + // not implemented + return 0; + } } } From 633c52d8482c438126565d95e1877bfa068dfebf Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:39:05 +0900 Subject: [PATCH 24/24] fix bug: don't always recreate RTCAudioDeviceModule --- modules/audio_device/audio_device_generic.cc | 1 + modules/audio_device/audio_device_impl.cc | 1 - .../api/peerconnection/RTCAudioDeviceModule.mm | 13 +++++++++++-- .../peerconnection/RTCPeerConnectionFactory.mm | 18 ++++++++---------- .../src/audio/audio_device_module_ios.mm | 3 ++- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/modules/audio_device/audio_device_generic.cc b/modules/audio_device/audio_device_generic.cc index 42eae8defc..0105c6ae24 100644 --- a/modules/audio_device/audio_device_generic.cc +++ b/modules/audio_device/audio_device_generic.cc @@ -64,6 +64,7 @@ int AudioDeviceGeneric::GetRecordAudioParameters( #endif // WEBRTC_IOS int32_t AudioDeviceGeneric::SetAudioDeviceSink(AudioDeviceSink* sink) { + RTC_LOG(LS_INFO) << __FUNCTION__ << "(" << sink << ")"; audio_device_module_sink_ = sink; return 0; } diff --git a/modules/audio_device/audio_device_impl.cc b/modules/audio_device/audio_device_impl.cc index 6f5fd10571..c45b9f3fef 100644 --- a/modules/audio_device/audio_device_impl.cc +++ b/modules/audio_device/audio_device_impl.cc @@ -946,7 +946,6 @@ int AudioDeviceModuleImpl::GetRecordAudioParameters( int32_t AudioDeviceModuleImpl::SetAudioDeviceSink(AudioDeviceSink* sink) const { RTC_LOG(LS_INFO) << __FUNCTION__ << "(" << sink << ")"; - CHECKinitialized_(); int32_t ok = audio_device_->SetAudioDeviceSink(sink); RTC_LOG(LS_INFO) << "output: " << ok; return ok; diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 8eebb20dca..dee10905f9 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -19,6 +19,7 @@ #import "RTCAudioDeviceModule.h" #import "RTCAudioDeviceModule+Private.h" #import "RTCIODevice+Private.h" +#import "base/RTCLogging.h" #import "sdk/objc/native/api/audio_device_module.h" @@ -27,6 +28,9 @@ AudioDeviceSink() {} void OnDevicesUpdated() override { + + RTCLogInfo(@"AudioDeviceSink OnDevicesUpdated"); + if (callback_handler_) { callback_handler_(); } @@ -44,12 +48,18 @@ @implementation RTC_OBJC_TYPE (RTCAudioDeviceModule) { - (instancetype)initWithNativeModule:(rtc::scoped_refptr )module workerThread:(rtc::Thread * )workerThread { + + RTCLogInfo(@"RTCAudioDeviceModule initWithNativeModule:workerThread:"); + self = [super init]; _native = module; _workerThread = workerThread; _sink = new AudioDeviceSink(); - _native->SetAudioDeviceSink(_sink); + + _workerThread->Invoke(RTC_FROM_HERE, [self] { + _native->SetAudioDeviceSink(_sink); + }); return self; } @@ -193,7 +203,6 @@ - (BOOL)initRecording { } - (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler { - _sink->callback_handler_ = handler; return YES; } diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index 15935cf549..b548312129 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -63,17 +63,13 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { std::unique_ptr _networkThread; std::unique_ptr _workerThread; std::unique_ptr _signalingThread; - rtc::scoped_refptr _audioDeviceModule; + rtc::scoped_refptr _nativeAudioDeviceModule; BOOL _hasStartedAecDump; } @synthesize nativeFactory = _nativeFactory; - -- (RTCAudioDeviceModule *)audioDeviceModule { - return [[RTCAudioDeviceModule alloc] initWithNativeModule: _audioDeviceModule - workerThread: _workerThread.get()]; -} +@synthesize audioDeviceModule = _audioDeviceModule; - (instancetype)init { #ifdef HAVE_NO_MEDIA @@ -225,16 +221,18 @@ - (instancetype)initWithNativeAudioEncoderFactory: dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); dependencies.trials = std::make_unique(); cricket::MediaEngineDependencies media_deps; - + // always create ADM on worker thread - auto adm = _workerThread->Invoke>(RTC_FROM_HERE, [&dependencies, &bypassVoiceProcessing]() { + _nativeAudioDeviceModule = _workerThread->Invoke>(RTC_FROM_HERE, [&dependencies, &bypassVoiceProcessing]() { return webrtc::AudioDeviceModule::Create(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, dependencies.task_queue_factory.get(), bypassVoiceProcessing == YES); }); - _audioDeviceModule = adm; - media_deps.adm = adm; + _audioDeviceModule = [[RTCAudioDeviceModule alloc] initWithNativeModule: _nativeAudioDeviceModule + workerThread: _workerThread.get()]; + + media_deps.adm = _nativeAudioDeviceModule; media_deps.task_queue_factory = dependencies.task_queue_factory.get(); media_deps.audio_encoder_factory = std::move(audioEncoderFactory); media_deps.audio_decoder_factory = std::move(audioDecoderFactory); diff --git a/sdk/objc/native/src/audio/audio_device_module_ios.mm b/sdk/objc/native/src/audio/audio_device_module_ios.mm index a7b1f680e8..9b620398db 100644 --- a/sdk/objc/native/src/audio/audio_device_module_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_module_ios.mm @@ -670,7 +670,8 @@ int32_t AudioDeviceModuleIOS::SetAudioDeviceSink(AudioDeviceSink* sink) const { // not implemented - return 0; + RTC_LOG(LS_WARNING) << __FUNCTION__ << "(" << sink << ") Not implemented"; + return -1; } } }