diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 5bf25ffc6d..1fc5bb4037 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -522,6 +522,8 @@ if (is_android) { sources = [ "api/org/webrtc/SoftwareVideoDecoderFactory.java", "api/org/webrtc/SoftwareVideoEncoderFactory.java", + "api/org/webrtc/SimulcastVideoEncoder.java", + "api/org/webrtc/SimulcastVideoEncoderFactory.java", ] deps = [ @@ -860,6 +862,23 @@ if (current_os == "linux" || is_android) { ] } + rtc_library("simulcast_jni") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "src/jni/simulcast_video_encoder.cc", + "src/jni/simulcast_video_encoder.h", + "src/jni/simulcast_video_encoder_factory.cc", + "src/jni/simulcast_video_encoder_factory.h" + ] + deps = [ + ":base_jni", + ":video_jni", + ":native_api_codecs", + "../../media:rtc_simulcast_encoder_adapter" + ] + } + rtc_library("libaom_av1_encoder_jni") { visibility = [ "*" ] allow_poison = [ "software_video_codecs" ] @@ -896,6 +915,7 @@ if (current_os == "linux" || is_android) { ":generated_swcodecs_jni", ":libvpx_vp8_jni", ":libvpx_vp9_jni", + ":simulcast_jni", ":native_api_jni", ":video_jni", "../../api/video_codecs:builtin_video_decoder_factory", diff --git a/sdk/android/api/org/webrtc/RtpParameters.java b/sdk/android/api/org/webrtc/RtpParameters.java index 9ca8311610..01f8082c57 100644 --- a/sdk/android/api/org/webrtc/RtpParameters.java +++ b/sdk/android/api/org/webrtc/RtpParameters.java @@ -76,6 +76,7 @@ public static class Encoding { // If non-null, scale the width and height down by this factor for video. If null, // implementation default scaling factor will be used. @Nullable public Double scaleResolutionDownBy; + @Nullable public String scalabilityMode; // SSRC to be used by this encoding. // Can't be changed between getParameters/setParameters. public Long ssrc; @@ -93,7 +94,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) { @CalledByNative("Encoding") Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority, Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate, - Integer numTemporalLayers, Double scaleResolutionDownBy, Long ssrc, + Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode, Long ssrc, boolean adaptiveAudioPacketTime) { this.rid = rid; this.active = active; @@ -104,6 +105,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) { this.maxFramerate = maxFramerate; this.numTemporalLayers = numTemporalLayers; this.scaleResolutionDownBy = scaleResolutionDownBy; + this.scalabilityMode = scalabilityMode; this.ssrc = ssrc; this.adaptiveAudioPacketTime = adaptiveAudioPacketTime; } @@ -160,6 +162,12 @@ Double getScaleResolutionDownBy() { return scaleResolutionDownBy; } + @Nullable + @CalledByNative("Encoding") + String getScalabilityMode() { + return scalabilityMode; + } + @CalledByNative("Encoding") Long getSsrc() { return ssrc; diff --git a/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java b/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java new file mode 100644 index 0000000000..da39a1b6a7 --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java @@ -0,0 +1,27 @@ +package org.webrtc; + +public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder { + + static native long nativeCreateEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info); + + VideoEncoderFactory primary; + VideoEncoderFactory fallback; + VideoCodecInfo info; + + public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) { + this.primary = primary; + this.fallback = fallback; + this.info = info; + } + + @Override + public long createNativeVideoEncoder() { + return nativeCreateEncoder(primary, fallback, info); + } + + @Override + public boolean isHardwareEncoder() { + return false; + } + +} diff --git a/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java new file mode 100644 index 0000000000..de1d8086fe --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Arrays; + +public class SimulcastVideoEncoderFactory implements VideoEncoderFactory { + + static native List nativeVP9Codecs(); + static native VideoCodecInfo nativeAV1Codec(); + + VideoEncoderFactory primary; + VideoEncoderFactory fallback; + + public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) { + this.primary = primary; + this.fallback = fallback; + } + + @Nullable + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + return new SimulcastVideoEncoder(primary, fallback, info); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + List codecs = new ArrayList(); + codecs.addAll(Arrays.asList(primary.getSupportedCodecs())); + if (fallback != null) { + codecs.addAll(Arrays.asList(fallback.getSupportedCodecs())); + } + codecs.addAll(nativeVP9Codecs()); + codecs.add(nativeAV1Codec()); + return codecs.toArray(new VideoCodecInfo[codecs.size()]); + } + +} diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java index 4f97cf74cf..86d67d6d5c 100644 --- a/sdk/android/api/org/webrtc/VideoCodecInfo.java +++ b/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -34,6 +34,7 @@ public class VideoCodecInfo { public final String name; public final Map params; + public int[] scalabilityModes; @Deprecated public final int payload; @CalledByNative @@ -41,6 +42,7 @@ public VideoCodecInfo(String name, Map params) { this.payload = 0; this.name = name; this.params = params; + this.scalabilityModes = new int[0]; } @Deprecated @@ -48,6 +50,7 @@ public VideoCodecInfo(int payload, String name, Map params) { this.payload = payload; this.name = name; this.params = params; + this.scalabilityModes = new int[0]; } @Override @@ -83,4 +86,16 @@ String getName() { Map getParams() { return params; } + + @CalledByNative + int[] getScalabilityModes() { + return scalabilityModes; + } + + @CalledByNative + void setScalabilityModes(int[] values) { + scalabilityModes = values; + } + + } diff --git a/sdk/android/src/jni/pc/rtp_parameters.cc b/sdk/android/src/jni/pc/rtp_parameters.cc index 4bd9ee0e1d..dfc7f17f6d 100644 --- a/sdk/android/src/jni/pc/rtp_parameters.cc +++ b/sdk/android/src/jni/pc/rtp_parameters.cc @@ -53,6 +53,7 @@ ScopedJavaLocalRef NativeToJavaRtpEncodingParameter( NativeToJavaInteger(env, encoding.max_framerate), NativeToJavaInteger(env, encoding.num_temporal_layers), NativeToJavaDouble(env, encoding.scale_resolution_down_by), + NativeToJavaString(env, encoding.scalability_mode), encoding.ssrc ? NativeToJavaLong(env, *encoding.ssrc) : nullptr, encoding.adaptive_ptime); } @@ -116,6 +117,11 @@ RtpEncodingParameters JavaToNativeRtpEncodingParameters( Java_Encoding_getScaleResolutionDownBy(jni, j_encoding_parameters); encoding.scale_resolution_down_by = JavaToNativeOptionalDouble(jni, j_scale_resolution_down_by); + ScopedJavaLocalRef j_scalability_mode = + Java_Encoding_getScalabilityMode(jni, j_encoding_parameters); + if (!IsNull(jni, j_scalability_mode)) { + encoding.scalability_mode = JavaToNativeString(jni, j_scalability_mode); + } encoding.adaptive_ptime = Java_Encoding_getAdaptivePTime(jni, j_encoding_parameters); ScopedJavaLocalRef j_ssrc = diff --git a/sdk/android/src/jni/simulcast_video_encoder.cc b/sdk/android/src/jni/simulcast_video_encoder.cc new file mode 100644 index 0000000000..18daf6f062 --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder.cc @@ -0,0 +1,34 @@ +#include + +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/video_encoder_factory_wrapper.h" +#include "sdk/android/src/jni/video_codec_info.h" +#include "sdk/android/native_api/codecs/wrapper.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "rtc_base/logging.h" + +using namespace webrtc; +using namespace webrtc::jni; + +#ifdef __cplusplus +extern "C" { +#endif + +// (VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) +JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jobject primary, jobject fallback, jobject info) { + RTC_LOG(LS_INFO) << "Create simulcast video encoder"; + JavaParamRef info_ref(info); + SdpVideoFormat format = VideoCodecInfoToSdpVideoFormat(env, info_ref); + + // TODO: 影響は軽微だが、リークする可能性があるので将来的に修正したい + // https://github.com/shiguredo-webrtc-build/webrtc-build/pull/16#pullrequestreview-600675795 + return NativeToJavaPointer(std::make_unique( + JavaToNativeVideoEncoderFactory(env, primary).release(), + fallback != nullptr ? JavaToNativeVideoEncoderFactory(env, fallback).release() : nullptr, + format).release()); +} + + +#ifdef __cplusplus +} +#endif diff --git a/sdk/android/src/jni/simulcast_video_encoder.h b/sdk/android/src/jni/simulcast_video_encoder.h new file mode 100644 index 0000000000..519be778e8 --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder.h @@ -0,0 +1,22 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_webrtc_SimulcastVideoEncoder */ + +#ifndef _Included_org_webrtc_SimulcastVideoEncoder +#define _Included_org_webrtc_SimulcastVideoEncoder +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_webrtc_SimulcastVideoEncoder + * Method: nativeCreateEncoder + * Signature: (Lorg/webrtc/VideoEncoderFactory;Lorg/webrtc/VideoEncoderFactory;Lorg/webrtc/VideoCodecInfo;)J + */ + +JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder + (JNIEnv *, jclass, jobject, jobject, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/sdk/android/src/jni/simulcast_video_encoder_factory.cc b/sdk/android/src/jni/simulcast_video_encoder_factory.cc new file mode 100644 index 0000000000..bc6ea5be6a --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder_factory.cc @@ -0,0 +1,40 @@ +#include + +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/video_encoder_factory_wrapper.h" +#include "sdk/android/src/jni/video_codec_info.h" +#include "sdk/android/native_api/codecs/wrapper.h" +#include "sdk/android/native_api/jni/class_loader.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "media/base/media_constants.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" + +using namespace webrtc; +using namespace webrtc::jni; + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeVP9Codecs + (JNIEnv *env, jclass klass) { + std::vector formats = SupportedVP9Codecs(true); + return NativeToJavaList(env, formats, &SdpVideoFormatToVideoCodecInfo).Release(); +} + +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeAV1Codec + (JNIEnv *env, jclass klass) { + SdpVideoFormat format = SdpVideoFormat( + cricket::kAv1CodecName, SdpVideoFormat::Parameters(), + LibaomAv1EncoderSupportedScalabilityModes()); + return SdpVideoFormatToVideoCodecInfo(env, format).Release(); +} + +#ifdef __cplusplus +} +#endif diff --git a/sdk/android/src/jni/simulcast_video_encoder_factory.h b/sdk/android/src/jni/simulcast_video_encoder_factory.h new file mode 100644 index 0000000000..d7426953a0 --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder_factory.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_webrtc_SimulcastVideoEncoderFactory */ + +#ifndef _Included_org_webrtc_SimulcastVideoEncoderFactory +#define _Included_org_webrtc_SimulcastVideoEncoderFactory +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_webrtc_SimulcastVideoEncoderFactory + * Method: nativeVP9Codecs + * Signature: ()Ljava/util/List; + */ +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeVP9Codecs + (JNIEnv *, jclass); + +/* + * Class: org_webrtc_SimulcastVideoEncoderFactory + * Method: nativeAV1Codec + * Signature: ()Lorg/webrtc/VideoCodecInfo; + */ +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeAV1Codec + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc index a218a1d23f..42e7b5d897 100644 --- a/sdk/android/src/jni/video_codec_info.cc +++ b/sdk/android/src/jni/video_codec_info.cc @@ -14,14 +14,22 @@ #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/jni_helpers.h" +#include "absl/container/inlined_vector.h" + namespace webrtc { namespace jni { SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni, const JavaRef& j_info) { + absl::InlinedVector scalabilityModes; + std::vector javaScalabilityModes = JavaToNativeIntArray(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info)); + for (const auto& scalabilityMode : javaScalabilityModes) { + scalabilityModes.push_back(static_cast(scalabilityMode)); + } return SdpVideoFormat( JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info)), - JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info))); + JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)), + scalabilityModes); } ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( @@ -29,8 +37,18 @@ ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( const SdpVideoFormat& format) { ScopedJavaLocalRef j_params = NativeToJavaStringMap(jni, format.parameters); - return Java_VideoCodecInfo_Constructor( + + ScopedJavaLocalRef codec = Java_VideoCodecInfo_Constructor( jni, NativeToJavaString(jni, format.name), j_params); + + size_t size = format.scalability_modes.size(); + std::vector temp(size); + for (size_t i = 0; i < size; i++) { + temp[i] = static_cast(format.scalability_modes[i]); + } + Java_VideoCodecInfo_setScalabilityModes(jni, codec, NativeToJavaIntArray(jni, temp)); + + return codec; } } // namespace jni