Merge pull request #2 from GetStream/patch/ios/simulcast

Support for simulcast in iOS SDK

sdk/BUILD.gn

@@ -230,9 +230,6 @@ if (is_ios || is_mac) {
           "objc/native/api/audio_device_module.h",
           "objc/native/api/audio_device_module.mm",
         ]
-        if (is_mac) {
-          frameworks = [ "AudioUnit.framework" ]
-        }
 
         deps = [
           ":audio_device",
@@ -293,9 +290,6 @@ if (is_ios || is_mac) {
         absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ]
 
         frameworks = [ "AudioToolbox.framework" ]
-        if (is_mac) {
-          frameworks += [ "AudioUnit.framework" ]
-        }
       }
 
       # This target exists to expose :audio_session_objc and
@@ -436,9 +430,6 @@ if (is_ios || is_mac) {
         "../rtc_base:threading",
         "../rtc_base:timeutils",
       ]
-      if (is_mac) {
-        frameworks = [ "AudioUnit.framework" ]
-      }
     }
 
     rtc_library("objc_audio_device_module") {
@@ -456,9 +447,6 @@ if (is_ios || is_mac) {
         "../modules/audio_device:audio_device_api",
         "../rtc_base:logging",
       ]
-      if (is_mac) {
-        frameworks = [ "AudioUnit.framework" ]
-      }
     }
 
     rtc_library("videosource_objc") {
@@ -741,6 +729,7 @@ if (is_ios || is_mac) {
       ]
 
       deps = [
+        ":simulcast",
         ":base_objc",
         ":native_video",
         ":videocodec_objc",
@@ -792,6 +781,22 @@ if (is_ios || is_mac) {
       ]
     }
 
+    rtc_library("simulcast") {
+      sources = [
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
+      ]
+
+      deps = [
+        ":base_objc",
+        ":wrapped_native_codec_objc",
+        "../media:rtc_media_base",
+        "../media:rtc_simulcast_encoder_adapter",
+      ]
+    }
+
     rtc_library("vp9") {
       visibility = [ "*" ]
       allow_poison = [ "software_video_codecs" ]
@@ -1304,6 +1309,7 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoDecoderH264.h",
           "objc/components/video_codec/RTCVideoEncoderFactoryH264.h",
           "objc/components/video_codec/RTCVideoEncoderH264.h",
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCCameraPreviewView.h",
           "objc/helpers/RTCDispatcher.h",
@@ -1351,6 +1357,7 @@ if (is_ios || is_mac) {
           "objc/api/video_codec/RTCVideoEncoderVP8.h",
           "objc/api/video_codec/RTCVideoEncoderVP9.h",
           "objc/api/video_codec/RTCVideoEncoderAV1.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
           "objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
           "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
         ]
@@ -1605,7 +1612,6 @@ if (is_ios || is_mac) {
         "../api/video:video_rtp_headers",
         "../api/video_codecs:video_codecs_api",
         "../common_video",
-        "../pc:video_track_source_proxy",
         "../rtc_base:buffer",
         "../rtc_base:logging",
         "../rtc_base:ssl",

sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h

@@ -69,6 +69,8 @@ RTC_OBJC_EXPORT
  https://w3c.github.io/webrtc-extensions/#dom-rtcrtpencodingparameters-adaptiveptime */
 @property(nonatomic, assign) BOOL adaptiveAudioPacketTime;
 
+@property(nonatomic, copy, nullable) NSString *scalabilityMode;
+
 - (instancetype)init;
 
 @end

sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm

@@ -25,6 +25,7 @@ @implementation RTC_OBJC_TYPE (RTCRtpEncodingParameters)
 @synthesize bitratePriority = _bitratePriority;
 @synthesize networkPriority = _networkPriority;
 @synthesize adaptiveAudioPacketTime = _adaptiveAudioPacketTime;
+@synthesize scalabilityMode = _scalabilityMode;
 
 - (instancetype)init {
   webrtc::RtpEncodingParameters nativeParameters;
@@ -59,6 +60,9 @@ - (instancetype)initWithNativeParameters:
     if (nativeParameters.ssrc) {
       _ssrc = [NSNumber numberWithUnsignedLong:*nativeParameters.ssrc];
     }
+    if (nativeParameters.scalability_mode.has_value()) {
+      _scalabilityMode = [NSString stringForStdString:nativeParameters.scalability_mode.value()];
+    }
     _bitratePriority = nativeParameters.bitrate_priority;
     _networkPriority = [RTC_OBJC_TYPE(RTCRtpEncodingParameters)
         priorityFromNativePriority:nativeParameters.network_priority];
@@ -96,6 +100,9 @@ - (instancetype)initWithNativeParameters:
   parameters.network_priority =
       [RTC_OBJC_TYPE(RTCRtpEncodingParameters) nativePriorityFromPriority:_networkPriority];
   parameters.adaptive_ptime = _adaptiveAudioPacketTime;
+  if (_scalabilityMode != nil) {
+    parameters.scalability_mode = [NSString stdStringForString:_scalabilityMode];
+  }
   return parameters;
 }
 

sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm

@@ -15,13 +15,24 @@
 @implementation RTC_OBJC_TYPE (RTCVideoCodecInfo)
 (Private)
 
-    - (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format {
+- (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format {
   NSMutableDictionary *params = [NSMutableDictionary dictionary];
   for (auto it = format.parameters.begin(); it != format.parameters.end(); ++it) {
     [params setObject:[NSString stringForStdString:it->second]
                forKey:[NSString stringForStdString:it->first]];
   }
-  return [self initWithName:[NSString stringForStdString:format.name] parameters:params];
+
+  NSMutableArray *scalabilityModes = [[NSMutableArray alloc] init];
+  if (!format.scalability_modes.empty()) {
+    for (const auto scalability_mode : format.scalability_modes) {
+      uint8_t value = static_cast<uint8_t>(scalability_mode);
+      [scalabilityModes addObject: [NSNumber numberWithUnsignedInt:value]];
+    }
+  }
+
+  return [self initWithName:[NSString stringForStdString:format.name]
+                 parameters:params
+           scalabilityModes:scalabilityModes];
 }
 
 - (webrtc::SdpVideoFormat)nativeSdpVideoFormat {
@@ -32,7 +43,13 @@ - (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format {
     parameters[key] = value;
   }
 
-  return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters);
+  absl::InlinedVector<webrtc::ScalabilityMode, webrtc::kScalabilityModeCount> scalabilityModes;
+  for (NSNumber *scalabilityMode in self.scalabilityModes) {
+    unsigned int value = [scalabilityMode unsignedIntValue];
+    scalabilityModes.push_back(static_cast<webrtc::ScalabilityMode>(value));
+  }
+
+  return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters, scalabilityModes);
 }
 
 @end

sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h

@@ -0,0 +1,13 @@
+#import "RTCMacros.h"
+#import "RTCVideoEncoder.h"
+#import "RTCVideoEncoderFactory.h"
+#import "RTCVideoCodecInfo.h"
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) : NSObject
+
++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                                                         fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
+                                                   videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo;
+
+@end

sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm

@@ -0,0 +1,26 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoEncoderSimulcast.h"
+#import "RTCWrappedNativeVideoEncoder.h"
+#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
+
+#include "native/api/video_encoder_factory.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+
+@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast)
+
++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                                                         fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
+                                                   videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo {
+    auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(primary);
+    auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(fallback);
+    auto nativeFormat = [videoCodecInfo nativeSdpVideoFormat];
+    return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc]
+        initWithNativeEncoder: std::make_unique<webrtc::SimulcastEncoderAdapter>(
+            nativePrimary.release(),
+            nativeFallback.release(),
+            std::move(nativeFormat))];
+}
+
+@end

sdk/objc/base/RTCVideoCodecInfo.h

@@ -14,6 +14,43 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
+typedef NS_ENUM(uint8_t, RTCScalabilityMode) {
+  RTCScalabilityModeL1T1,
+  RTCScalabilityModeL1T2,
+  RTCScalabilityModeL1T3,
+  RTCScalabilityModeL2T1,
+  RTCScalabilityModeL2T1h,
+  RTCScalabilityModeL2T1_KEY,
+  RTCScalabilityModeL2T2,
+  RTCScalabilityModeL2T2h,
+  RTCScalabilityModeL2T2_KEY,
+  RTCScalabilityModeL2T2_KEY_SHIFT,
+  RTCScalabilityModeL2T3,
+  RTCScalabilityModeL2T3h,
+  RTCScalabilityModeL2T3_KEY,
+  RTCScalabilityModeL3T1,
+  RTCScalabilityModeL3T1h,
+  RTCScalabilityModeL3T1_KEY,
+  RTCScalabilityModeL3T2,
+  RTCScalabilityModeL3T2h,
+  RTCScalabilityModeL3T2_KEY,
+  RTCScalabilityModeL3T3,
+  RTCScalabilityModeL3T3h,
+  RTCScalabilityModeL3T3_KEY,
+  RTCScalabilityModeS2T1,
+  RTCScalabilityModeS2T1h,
+  RTCScalabilityModeS2T2,
+  RTCScalabilityModeS2T2h,
+  RTCScalabilityModeS2T3,
+  RTCScalabilityModeS2T3h,
+  RTCScalabilityModeS3T1,
+  RTCScalabilityModeS3T1h,
+  RTCScalabilityModeS3T2,
+  RTCScalabilityModeS3T2h,
+  RTCScalabilityModeS3T3,
+  RTCScalabilityModeS3T3h,
+};
+
 /** Holds information to identify a codec. Corresponds to webrtc::SdpVideoFormat. */
 RTC_OBJC_EXPORT
 @interface RTC_OBJC_TYPE (RTCVideoCodecInfo) : NSObject <NSCoding>
@@ -22,14 +59,18 @@ RTC_OBJC_EXPORT
 
 - (instancetype)initWithName:(NSString *)name;
 
+- (instancetype)initWithName:(NSString *)name
+                  parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters;
 - (instancetype)initWithName:(NSString *)name
                   parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters
+            scalabilityModes:(nullable NSArray<NSNumber *> *)scalabilityModes
     NS_DESIGNATED_INITIALIZER;
 
 - (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info;
 
 @property(nonatomic, readonly) NSString *name;
 @property(nonatomic, readonly) NSDictionary<NSString *, NSString *> *parameters;
+@property(nonatomic, readonly) NSArray<NSNumber *> *scalabilityModes;
 
 @end
 

sdk/objc/base/RTCVideoCodecInfo.m

@@ -14,21 +14,29 @@ @implementation RTC_OBJC_TYPE (RTCVideoCodecInfo)
 
 @synthesize name = _name;
 @synthesize parameters = _parameters;
+@synthesize scalabilityModes = _scalabilityModes;
 
 - (instancetype)initWithName:(NSString *)name {
   return [self initWithName:name parameters:nil];
 }
 
 - (instancetype)initWithName:(NSString *)name
                   parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters {
+  return [self initWithName:name parameters:parameters scalabilityModes:nil];
+}
+
+- (instancetype)initWithName:(NSString *)name
+                  parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters
+            scalabilityModes:(nullable NSArray<NSNumber *> *)scalabilityModes {
   if (self = [super init]) {
     _name = name;
     _parameters = (parameters ? parameters : @{});
+    _scalabilityModes = (scalabilityModes ? scalabilityModes : @[]);
   }
-
   return self;
 }
 
+
 - (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info {
   if (!info ||
       ![self.name isEqualToString:info.name] ||

sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h

@@ -0,0 +1,16 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoEncoderFactory.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)>
+
+- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                       fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm

@@ -0,0 +1,63 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoCodecInfo.h"
+#import "RTCVideoEncoderFactorySimulcast.h"
+#import "api/video_codec/RTCVideoEncoderSimulcast.h"
+#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
+
+#include "absl/container/inlined_vector.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_codec.h"
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+#include "modules/video_coding/codecs/vp9/include/vp9.h"
+#include "media/base/media_constants.h"
+
+@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) ()
+
+@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> primary;
+@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> fallback;
+
+@end
+
+
+@implementation RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast)
+
+@synthesize primary = _primary;
+@synthesize fallback = _fallback;
+
+- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                       fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback {
+    if (self = [super init]) {
+        _primary = primary;
+        _fallback = fallback;
+    }
+    return self;
+}
+
+- (nullable id<RTC_OBJC_TYPE(RTCVideoEncoder)>)createEncoder: (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info {
+    return [RTCVideoEncoderSimulcast simulcastEncoderWithPrimary: _primary fallback: _fallback videoCodecInfo: info];
+}
+
+- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs {
+    NSArray *supportedCodecs = [[_primary supportedCodecs] arrayByAddingObjectsFromArray: [_fallback supportedCodecs]];
+
+    NSMutableArray<RTCVideoCodecInfo *> *addingCodecs = [[NSMutableArray alloc] init];
+
+    for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs(true)) {
+        RTCVideoCodecInfo *codec = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat: format];
+        [addingCodecs addObject: codec];
+    }
+
+    auto av1Format = webrtc::SdpVideoFormat(
+         cricket::kAv1CodecName, webrtc::SdpVideoFormat::Parameters(),
+         webrtc::LibaomAv1EncoderSupportedScalabilityModes());
+    RTCVideoCodecInfo *av1Codec = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat: av1Format];
+    [addingCodecs addObject: av1Codec];
+
+    return [supportedCodecs arrayByAddingObjectsFromArray: addingCodecs];
+}
+
+
+@end