Merge to M93: Change how alignment requirements are processed

Software fallback wrapper now reports least common multiple of requirements for two encoders. SimulcastEncoderAdapter queries actual encoder before InitEncode call and requests alignment for all layers if simulcast is not supported by any of the encoders. (cherry picked from commit c98aebb) Bug: chromium:1084702 Change-Id: Iaed8190737125d447036b6c664b863be72556a5d Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/225881 Reviewed-by: Niels Moller <nisse@webrtc.org> Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org> Cr-Original-Commit-Position: refs/heads/master@{#34466} Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228561 Reviewed-by: Åsa Persson <asapersson@webrtc.org> Cr-Commit-Position: refs/branch-heads/4577@{#5} Cr-Branched-From: 5196931-refs/heads/master@{#34463}
webrtc-sdk · Aug 13, 2021 · 2afcec3 · 2afcec3
1 parent 05562e6
commit 2afcec3
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 17 deletions.
diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc
@@ -25,6 +25,7 @@
 #include "api/video/video_frame.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
+#include "media/base/video_common.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "modules/video_coding/utility/simulcast_utility.h"
 #include "rtc_base/checks.h"
@@ -417,6 +418,13 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
   EncoderInfo info =
       IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
 
+  info.requested_resolution_alignment = cricket::LeastCommonMultiple(
+      fallback_encoder_info.requested_resolution_alignment,
+      default_encoder_info.requested_resolution_alignment);
+  info.apply_alignment_to_all_simulcast_layers =
+      fallback_encoder_info.apply_alignment_to_all_simulcast_layers ||
+      default_encoder_info.apply_alignment_to_all_simulcast_layers;
+
   if (fallback_params_.has_value()) {
     const auto settings = (encoder_state_ == EncoderState::kForcedFallback)
                               ? fallback_encoder_info.scaling_settings

diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc
@@ -149,9 +149,13 @@ namespace webrtc {
 
 SimulcastEncoderAdapter::EncoderContext::EncoderContext(
     std::unique_ptr<VideoEncoder> encoder,
-    bool prefer_temporal_support)
+    bool prefer_temporal_support,
+    VideoEncoder::EncoderInfo primary_info,
+    VideoEncoder::EncoderInfo fallback_info)
     : encoder_(std::move(encoder)),
-      prefer_temporal_support_(prefer_temporal_support) {}
+      prefer_temporal_support_(prefer_temporal_support),
+      primary_info_(std::move(primary_info)),
+      fallback_info_(std::move(fallback_info)) {}
 
 void SimulcastEncoderAdapter::EncoderContext::Release() {
   if (encoder_) {
@@ -690,7 +694,7 @@ void SimulcastEncoderAdapter::DestroyStoredEncoders() {
 
 std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
 SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
-    bool is_lowest_quality_stream) {
+    bool is_lowest_quality_stream) const {
   bool prefer_temporal_support = fallback_encoder_factory_ != nullptr &&
                                  is_lowest_quality_stream &&
                                  prefer_temporal_support_on_base_layer_;
@@ -712,14 +716,20 @@ SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
   } else {
     std::unique_ptr<VideoEncoder> encoder =
         primary_encoder_factory_->CreateVideoEncoder(video_format_);
+    VideoEncoder::EncoderInfo primary_info = encoder->GetEncoderInfo();
+    VideoEncoder::EncoderInfo fallback_info = primary_info;
     if (fallback_encoder_factory_ != nullptr) {
+      std::unique_ptr<VideoEncoder> fallback_encoder =
+          fallback_encoder_factory_->CreateVideoEncoder(video_format_);
+      fallback_info = fallback_encoder->GetEncoderInfo();
       encoder = CreateVideoEncoderSoftwareFallbackWrapper(
-          fallback_encoder_factory_->CreateVideoEncoder(video_format_),
-          std::move(encoder), prefer_temporal_support);
+          std::move(fallback_encoder), std::move(encoder),
+          prefer_temporal_support);
     }
 
     encoder_context = std::make_unique<SimulcastEncoderAdapter::EncoderContext>(
-        std::move(encoder), prefer_temporal_support);
+        std::move(encoder), prefer_temporal_support, primary_info,
+        fallback_info);
   }
 
   encoder_context->encoder().RegisterEncodeCompleteCallback(
@@ -789,9 +799,11 @@ webrtc::VideoCodec SimulcastEncoderAdapter::MakeStreamCodec(
 void SimulcastEncoderAdapter::OverrideFromFieldTrial(
     VideoEncoder::EncoderInfo* info) const {
   if (encoder_info_override_.requested_resolution_alignment()) {
-    info->requested_resolution_alignment =
-        *encoder_info_override_.requested_resolution_alignment();
+    info->requested_resolution_alignment = cricket::LeastCommonMultiple(
+        info->requested_resolution_alignment,
+        *encoder_info_override_.requested_resolution_alignment());
     info->apply_alignment_to_all_simulcast_layers =
+        info->apply_alignment_to_all_simulcast_layers ||
         encoder_info_override_.apply_alignment_to_all_simulcast_layers();
   }
   if (!encoder_info_override_.resolution_bitrate_limits().empty()) {
@@ -815,7 +827,34 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
   encoder_info.apply_alignment_to_all_simulcast_layers = false;
   encoder_info.supports_native_handle = true;
   encoder_info.scaling_settings.thresholds = absl::nullopt;
+
   if (stream_contexts_.empty()) {
+    // GetEncoderInfo queried before InitEncode. Only alignment info is needed
+    // to be filled.
+    // Create one encoder and query it.
+
+    std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context =
+        FetchOrCreateEncoderContext(true);
+
+    const VideoEncoder::EncoderInfo& primary_info =
+        encoder_context->PrimaryInfo();
+    const VideoEncoder::EncoderInfo& fallback_info =
+        encoder_context->FallbackInfo();
+
+    encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(
+        primary_info.requested_resolution_alignment,
+        fallback_info.requested_resolution_alignment);
+
+    encoder_info.apply_alignment_to_all_simulcast_layers =
+        primary_info.apply_alignment_to_all_simulcast_layers ||
+        fallback_info.apply_alignment_to_all_simulcast_layers;
+
+    if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) {
+      encoder_info.apply_alignment_to_all_simulcast_layers = true;
+    }
+
+    cached_encoder_contexts_.emplace_back(std::move(encoder_context));
+
     OverrideFromFieldTrial(&encoder_info);
     return encoder_info;
   }
@@ -825,7 +864,6 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
   for (size_t i = 0; i < stream_contexts_.size(); ++i) {
     VideoEncoder::EncoderInfo encoder_impl_info =
         stream_contexts_[i].encoder().GetEncoderInfo();
-
     if (i == 0) {
       // Encoder name indicates names of all sub-encoders.
       encoder_info.implementation_name += " (";
@@ -864,7 +902,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
     encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(
         encoder_info.requested_resolution_alignment,
         encoder_impl_info.requested_resolution_alignment);
-    if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) {
+    // request alignment on all layers if any of the encoders may need it, or
+    // if any non-top layer encoder requests a non-trivial alignment.
+    if (encoder_impl_info.apply_alignment_to_all_simulcast_layers ||
+        (encoder_impl_info.requested_resolution_alignment > 1 &&
+         (codec_.simulcastStream[i].height < codec_.height ||
+          codec_.simulcastStream[i].width < codec_.width))) {
       encoder_info.apply_alignment_to_all_simulcast_layers = true;
     }
   }

diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h
@@ -71,16 +71,24 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
   class EncoderContext {
    public:
     EncoderContext(std::unique_ptr<VideoEncoder> encoder,
-                   bool prefer_temporal_support);
+                   bool prefer_temporal_support,
+                   VideoEncoder::EncoderInfo primary_info,
+                   VideoEncoder::EncoderInfo fallback_info);
     EncoderContext& operator=(EncoderContext&&) = delete;
 
     VideoEncoder& encoder() { return *encoder_; }
     bool prefer_temporal_support() { return prefer_temporal_support_; }
     void Release();
 
+    const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; }
+
+    const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; }
+
    private:
     std::unique_ptr<VideoEncoder> encoder_;
     bool prefer_temporal_support_;
+    const VideoEncoder::EncoderInfo primary_info_;
+    const VideoEncoder::EncoderInfo fallback_info_;
   };
 
   class StreamContext : public EncodedImageCallback {
@@ -138,8 +146,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
 
   void DestroyStoredEncoders();
 
+  // This method creates encoder. May reuse previously created encoders from
+  // |cached_encoder_contexts_|. It's const because it's used from
+  // const GetEncoderInfo().
   std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
-      bool is_lowest_quality_stream);
+      bool is_lowest_quality_stream) const;
 
   webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
                                      int stream_idx,
@@ -169,9 +180,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
   // Used for checking the single-threaded access of the encoder interface.
   RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
 
-  // Store encoders in between calls to Release and InitEncode, so they don't
-  // have to be recreated. Remaining encoders are destroyed by the destructor.
-  std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
+  // Store previously created and released encoders , so they don't have to be
+  // recreated. Remaining encoders are destroyed by the destructor.
+  // Marked as |mutable| becuase we may need to temporarily create encoder in
+  // GetEncoderInfo(), which is const.
+  mutable std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
 
   const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
   const bool boost_base_layer_quality_;

diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc
@@ -900,8 +900,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) {
 }
 
 TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) {
-  EXPECT_EQ("SimulcastEncoderAdapter",
-            adapter_->GetEncoderInfo().implementation_name);
   SimulcastTestFixtureImpl::DefaultSettings(
       &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
       kVideoCodecVP8);
@@ -910,6 +908,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) {
   encoder_names.push_back("codec2");
   encoder_names.push_back("codec3");
   helper_->factory()->SetEncoderNames(encoder_names);
+  EXPECT_EQ("SimulcastEncoderAdapter",
+            adapter_->GetEncoderInfo().implementation_name);
   EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
   EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)",
             adapter_->GetEncoderInfo().implementation_name);