Support for simulcast in Android SDK (#4)

Co-authored-by: kanat <>

sdk/android/BUILD.gn

@@ -522,6 +522,8 @@ if (is_android) {
     sources = [
       "api/org/webrtc/SoftwareVideoDecoderFactory.java",
       "api/org/webrtc/SoftwareVideoEncoderFactory.java",
+      "api/org/webrtc/SimulcastVideoEncoder.java",
+      "api/org/webrtc/SimulcastVideoEncoderFactory.java",
     ]
 
     deps = [
@@ -860,6 +862,23 @@ if (current_os == "linux" || is_android) {
     ]
   }
 
+  rtc_library("simulcast_jni") {
+    visibility = [ "*" ]
+    allow_poison = [ "software_video_codecs" ]
+    sources = [
+      "src/jni/simulcast_video_encoder.cc",
+      "src/jni/simulcast_video_encoder.h",
+      "src/jni/simulcast_video_encoder_factory.cc",
+      "src/jni/simulcast_video_encoder_factory.h"
+    ]
+    deps = [
+      ":base_jni",
+      ":video_jni",
+      ":native_api_codecs",
+      "../../media:rtc_simulcast_encoder_adapter"
+    ]
+  }
+
   rtc_library("libaom_av1_encoder_jni") {
     visibility = [ "*" ]
     allow_poison = [ "software_video_codecs" ]
@@ -896,6 +915,7 @@ if (current_os == "linux" || is_android) {
       ":generated_swcodecs_jni",
       ":libvpx_vp8_jni",
       ":libvpx_vp9_jni",
+      ":simulcast_jni",
       ":native_api_jni",
       ":video_jni",
       "../../api/video_codecs:builtin_video_decoder_factory",

sdk/android/api/org/webrtc/RtpParameters.java

@@ -76,6 +76,7 @@ public static class Encoding {
     // If non-null, scale the width and height down by this factor for video. If null,
     // implementation default scaling factor will be used.
     @Nullable public Double scaleResolutionDownBy;
+    @Nullable public String scalabilityMode;
     // SSRC to be used by this encoding.
     // Can't be changed between getParameters/setParameters.
     public Long ssrc;
@@ -93,7 +94,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
     @CalledByNative("Encoding")
     Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority,
         Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate,
-        Integer numTemporalLayers, Double scaleResolutionDownBy, Long ssrc,
+        Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode, Long ssrc,
         boolean adaptiveAudioPacketTime) {
       this.rid = rid;
       this.active = active;
@@ -104,6 +105,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
       this.maxFramerate = maxFramerate;
       this.numTemporalLayers = numTemporalLayers;
       this.scaleResolutionDownBy = scaleResolutionDownBy;
+      this.scalabilityMode = scalabilityMode;
       this.ssrc = ssrc;
       this.adaptiveAudioPacketTime = adaptiveAudioPacketTime;
     }
@@ -160,6 +162,12 @@ Double getScaleResolutionDownBy() {
       return scaleResolutionDownBy;
     }
 
+    @Nullable
+    @CalledByNative("Encoding")
+    String getScalabilityMode() {
+      return scalabilityMode;
+    }
+
     @CalledByNative("Encoding")
     Long getSsrc() {
       return ssrc;

sdk/android/api/org/webrtc/SimulcastVideoEncoder.java

@@ -0,0 +1,27 @@
+package org.webrtc;
+
+public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder {
+
+    static native long nativeCreateEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info);
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+    VideoCodecInfo info;
+
+    public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) {
+        this.primary = primary;
+        this.fallback = fallback;
+        this.info = info;
+    }
+
+    @Override
+    public long createNativeVideoEncoder() {
+        return nativeCreateEncoder(primary, fallback, info);
+    }
+
+    @Override
+    public boolean isHardwareEncoder() {
+        return false;
+    }
+
+}

sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import androidx.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Arrays;
+
+public class SimulcastVideoEncoderFactory implements VideoEncoderFactory {
+
+    static native List<VideoCodecInfo> nativeVP9Codecs();
+    static native VideoCodecInfo nativeAV1Codec();
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+
+    public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) {
+        this.primary = primary;
+        this.fallback = fallback;
+    }
+
+    @Nullable
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo info) {
+        return new SimulcastVideoEncoder(primary, fallback, info);
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
+        codecs.addAll(Arrays.asList(primary.getSupportedCodecs()));
+        if (fallback != null) {
+            codecs.addAll(Arrays.asList(fallback.getSupportedCodecs()));
+        }
+        codecs.addAll(nativeVP9Codecs());
+        codecs.add(nativeAV1Codec());
+        return codecs.toArray(new VideoCodecInfo[codecs.size()]);
+    }
+
+}

sdk/android/api/org/webrtc/VideoCodecInfo.java

@@ -34,20 +34,23 @@ public class VideoCodecInfo {
 
   public final String name;
   public final Map<String, String> params;
+  public int[] scalabilityModes;
   @Deprecated public final int payload;
 
   @CalledByNative
   public VideoCodecInfo(String name, Map<String, String> params) {
     this.payload = 0;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = new int[0];
   }
 
   @Deprecated
   public VideoCodecInfo(int payload, String name, Map<String, String> params) {
     this.payload = payload;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = new int[0];
   }
 
   @Override
@@ -83,4 +86,16 @@ String getName() {
   Map getParams() {
     return params;
   }
+
+  @CalledByNative
+  int[] getScalabilityModes() {
+    return scalabilityModes;
+  }
+
+  @CalledByNative
+  void setScalabilityModes(int[] values) {
+    scalabilityModes = values;
+  }
+
+
 }

sdk/android/src/jni/pc/rtp_parameters.cc

@@ -53,6 +53,7 @@ ScopedJavaLocalRef<jobject> NativeToJavaRtpEncodingParameter(
       NativeToJavaInteger(env, encoding.max_framerate),
       NativeToJavaInteger(env, encoding.num_temporal_layers),
       NativeToJavaDouble(env, encoding.scale_resolution_down_by),
+      NativeToJavaString(env, encoding.scalability_mode),
       encoding.ssrc ? NativeToJavaLong(env, *encoding.ssrc) : nullptr,
       encoding.adaptive_ptime);
 }
@@ -116,6 +117,11 @@ RtpEncodingParameters JavaToNativeRtpEncodingParameters(
       Java_Encoding_getScaleResolutionDownBy(jni, j_encoding_parameters);
   encoding.scale_resolution_down_by =
       JavaToNativeOptionalDouble(jni, j_scale_resolution_down_by);
+  ScopedJavaLocalRef<jstring> j_scalability_mode =
+      Java_Encoding_getScalabilityMode(jni, j_encoding_parameters);
+  if (!IsNull(jni, j_scalability_mode)) {
+    encoding.scalability_mode = JavaToNativeString(jni, j_scalability_mode);
+  }
   encoding.adaptive_ptime =
       Java_Encoding_getAdaptivePTime(jni, j_encoding_parameters);
   ScopedJavaLocalRef<jobject> j_ssrc =

sdk/android/src/jni/simulcast_video_encoder.cc

@@ -0,0 +1,34 @@
+#include <jni.h>
+
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_encoder_factory_wrapper.h"
+#include "sdk/android/src/jni/video_codec_info.h"
+#include "sdk/android/native_api/codecs/wrapper.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+#include "rtc_base/logging.h"
+
+using namespace webrtc;
+using namespace webrtc::jni;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// (VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info)
+JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jobject primary, jobject fallback, jobject info) {
+    RTC_LOG(LS_INFO) << "Create simulcast video encoder";
+    JavaParamRef<jobject> info_ref(info);
+    SdpVideoFormat format = VideoCodecInfoToSdpVideoFormat(env, info_ref);
+
+    // TODO: 影響は軽微だが、リークする可能性があるので将来的に修正したい
+    // https://github.com/shiguredo-webrtc-build/webrtc-build/pull/16#pullrequestreview-600675795
+    return NativeToJavaPointer(std::make_unique<SimulcastEncoderAdapter>(
+			    JavaToNativeVideoEncoderFactory(env, primary).release(),
+			    fallback != nullptr ? JavaToNativeVideoEncoderFactory(env, fallback).release() : nullptr,
+			    format).release());
+}
+
+
+#ifdef __cplusplus
+}
+#endif

sdk/android/src/jni/simulcast_video_encoder_factory.cc

@@ -0,0 +1,40 @@
+#include <jni.h>
+
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_encoder_factory_wrapper.h"
+#include "sdk/android/src/jni/video_codec_info.h"
+#include "sdk/android/native_api/codecs/wrapper.h"
+#include "sdk/android/native_api/jni/class_loader.h"
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+#include "modules/video_coding/codecs/vp9/include/vp9.h"
+#include "media/base/media_constants.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+#include "absl/container/inlined_vector.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_codec.h"
+
+using namespace webrtc;
+using namespace webrtc::jni;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeVP9Codecs
+  (JNIEnv *env, jclass klass) {
+    std::vector<SdpVideoFormat> formats = SupportedVP9Codecs(true);
+    return NativeToJavaList(env, formats, &SdpVideoFormatToVideoCodecInfo).Release();
+}
+
+JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeAV1Codec
+  (JNIEnv *env, jclass klass) {
+    SdpVideoFormat format = SdpVideoFormat(
+        cricket::kAv1CodecName, SdpVideoFormat::Parameters(),
+        LibaomAv1EncoderSupportedScalabilityModes());
+    return SdpVideoFormatToVideoCodecInfo(env, format).Release();
+}
+
+#ifdef __cplusplus
+}
+#endif

sdk/android/src/jni/video_codec_info.cc

@@ -14,23 +14,41 @@
 #include "sdk/android/native_api/jni/java_types.h"
 #include "sdk/android/src/jni/jni_helpers.h"
 
+#include "absl/container/inlined_vector.h"
+
 namespace webrtc {
 namespace jni {
 
 SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni,
                                               const JavaRef<jobject>& j_info) {
+  absl::InlinedVector<webrtc::ScalabilityMode, webrtc::kScalabilityModeCount> scalabilityModes;
+  std::vector<int32_t> javaScalabilityModes = JavaToNativeIntArray(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info));
+  for (const auto& scalabilityMode : javaScalabilityModes) {
+    scalabilityModes.push_back(static_cast<webrtc::ScalabilityMode>(scalabilityMode));
+  }
   return SdpVideoFormat(
       JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info)),
-      JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)));
+      JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)),
+      scalabilityModes);
 }
 
 ScopedJavaLocalRef<jobject> SdpVideoFormatToVideoCodecInfo(
     JNIEnv* jni,
     const SdpVideoFormat& format) {
   ScopedJavaLocalRef<jobject> j_params =
       NativeToJavaStringMap(jni, format.parameters);
-  return Java_VideoCodecInfo_Constructor(
+
+  ScopedJavaLocalRef<jobject> codec = Java_VideoCodecInfo_Constructor(
       jni, NativeToJavaString(jni, format.name), j_params);
+
+  size_t size = format.scalability_modes.size();
+  std::vector<int32_t> temp(size);
+  for (size_t i = 0; i < size; i++) {
+    temp[i] = static_cast<jint>(format.scalability_modes[i]);
+  }
+  Java_VideoCodecInfo_setScalabilityModes(jni, codec, NativeToJavaIntArray(jni, temp));
+
+  return codec;
 }
 
 }  // namespace jni