enalbing low latency mode for RTC

api/bitrate_constraints.h

@@ -23,7 +23,7 @@ struct BitrateConstraints {
   int max_bitrate_bps = -1;
 
  private:
-  static constexpr int kDefaultStartBitrateBps = 300000;
+  static constexpr int kDefaultStartBitrateBps = 5000000;  // Updated from 300k to 5MB for low latency mode.
 };
 
 // Like std::min, but considers non-positive values to be unset.

api/rtp_headers.h

@@ -127,6 +127,9 @@ struct RTPHeaderExtension {
   // For identifying the media section used to interpret this RTP packet. See
   // https://tools.ietf.org/html/draft-ietf-mmusic-sdp-bundle-negotiation-38
   Mid mid;
+
+  // Required for Temporal scalability support
+  uint16_t picture_id;
 };
 
 struct RTPHeader {

api/rtpparameters.cc

@@ -132,6 +132,9 @@ const int RtpExtension::kMidDefaultId = 9;
 const char RtpExtension::kEncryptHeaderExtensionsUri[] =
     "urn:ietf:params:rtp-hdrext:encrypt";
 
+const char RtpExtension::kPictureIdUri[] = "urn:intel:picture-id";
+const int RtpExtension::kPictureIdDefaultId = 12;
+
 const int RtpExtension::kMinId = 1;
 const int RtpExtension::kMaxId = 14;
 
@@ -149,7 +152,8 @@ bool RtpExtension::IsSupportedForVideo(const std::string& uri) {
          uri == webrtc::RtpExtension::kPlayoutDelayUri ||
          uri == webrtc::RtpExtension::kVideoContentTypeUri ||
          uri == webrtc::RtpExtension::kVideoTimingUri ||
-         uri == webrtc::RtpExtension::kMidUri;
+         uri == webrtc::RtpExtension::kMidUri ||
+         uri == webrtc::RtpExtension::kPictureIdUri;
 }
 
 bool RtpExtension::IsEncryptionSupported(const std::string& uri) {

api/rtpparameters.h

@@ -299,6 +299,10 @@ struct RtpExtension {
   // https://tools.ietf.org/html/rfc6904
   static const char kEncryptHeaderExtensionsUri[];
 
+  // Picture ID for temporal scalability support
+  static const char kPictureIdUri[];
+  static const int kPictureIdDefaultId;
+
   // Inclusive min and max IDs for one-byte header extensions, per RFC5285.
   static const int kMinId;
   static const int kMaxId;

call/call_config.h

@@ -26,7 +26,7 @@ struct CallConfig {
   explicit CallConfig(RtcEventLog* event_log);
   ~CallConfig();
 
-  RTC_DEPRECATED static constexpr int kDefaultStartBitrateBps = 300000;
+  RTC_DEPRECATED static constexpr int kDefaultStartBitrateBps = 5000000;  //Updated from 300k to 5MB for low latency
 
   // Bitrate config used until valid bitrate estimates are calculated. Also
   // used to cap total bitrate used. This comes from the remote connection.

call/rtp_payload_params.cc

@@ -76,6 +76,7 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
       h264_header.packetization_mode =
           info.codecSpecific.H264.packetization_mode;
       rtp->simulcastIdx = info.codecSpecific.H264.simulcast_idx;
+      rtp->picture_id = info.codecSpecific.H264.picture_id;
       return;
     }
     case kVideoCodecMultiplex:

call/rtp_transport_controller_send.cc

@@ -23,13 +23,19 @@ namespace webrtc {
 namespace {
 static const int64_t kRetransmitWindowSizeMs = 500;
 const char kTaskQueueExperiment[] = "WebRTC-TaskQueueCongestionControl";
+const char kLowLatencyStreaming[] = "OWT-LowLatencyMode";
 using TaskQueueController = webrtc::webrtc_cc::SendSideCongestionController;
 
 bool TaskQueueExperimentEnabled() {
   std::string trial = webrtc::field_trial::FindFullName(kTaskQueueExperiment);
   return trial.find("Enable") == 0;
 }
 
+bool LowLatencyStreamingEnabled() {
+  std::string trial = webrtc::field_trial::FindFullName(kLowLatencyStreaming);
+  return trial.find("Enabled") == 0;
+}
+
 std::unique_ptr<SendSideCongestionControllerInterface> CreateController(
     Clock* clock,
     rtc::TaskQueue* task_queue,
@@ -75,7 +81,11 @@ RtpTransportControllerSend::RtpTransportControllerSend(
 
   process_thread_->RegisterModule(&pacer_, RTC_FROM_HERE);
   process_thread_->RegisterModule(send_side_cc_.get(), RTC_FROM_HERE);
-  process_thread_->Start();
+  if (!LowLatencyStreamingEnabled()) {
+    process_thread_->Start();
+  } else {
+    process_thread_->StartWithHighPriority();
+  }
 }
 
 RtpTransportControllerSend::~RtpTransportControllerSend() {

media/engine/webrtcvoiceengine.cc

@@ -298,6 +298,15 @@ void WebRtcVoiceEngine::Init() {
     options.experimental_ns = false;
     options.intelligibility_enhancer = false;
     options.residual_echo_detector = true;
+    // Overrided for low latency mode
+    if (webrtc::field_trial::IsEnabled("OWT-LowLatencyMode")) {
+      options.echo_cancellation = false;
+      options.auto_gain_control = false;
+      options.noise_suppression = false;
+      options.highpass_filter = false;
+      options.typing_detection = false;
+      options.residual_echo_detector = false;
+    }
     bool error = ApplyOptions(options);
     RTC_DCHECK(error);
   }
@@ -606,6 +615,10 @@ RtpCapabilities WebRtcVoiceEngine::GetCapabilities() const {
   // demuxing is completed.
   // capabilities.header_extensions.push_back(webrtc::RtpExtension(
   //     webrtc::RtpExtension::kMidUri, webrtc::RtpExtension::kMidDefaultId));
+  // Temporal scalability support.
+  capabilities.header_extensions.push_back(
+      webrtc::RtpExtension(webrtc::RtpExtension::kPictureIdUri,
+                           webrtc::RtpExtension::kPictureIdDefaultId));
   return capabilities;
 }
 

modules/audio_coding/codecs/opus/audio_encoder_opus.cc

@@ -143,6 +143,13 @@ int CalculateBitrate(int max_playback_rate_hz,
   const int default_bitrate =
       CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
 
+  // For lown latency mode we fix the audio bitrate.
+  if (field_trial::IsEnabled("OWT-LowLatencyMode")) {
+    if (default_bitrate != -1)
+      return 256000;
+    return default_bitrate;
+  }
+
   if (bitrate_param) {
     const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
     if (bitrate) {

modules/bitrate_controller/send_side_bandwidth_estimation.cc

@@ -195,6 +195,9 @@ void SendSideBandwidthEstimation::CurrentEstimate(int* bitrate,
 
 void SendSideBandwidthEstimation::UpdateReceiverEstimate(int64_t now_ms,
                                                          uint32_t bandwidth) {
+  // At low latency mode we don't responde to BWE.
+  if (field_trial::IsEnabled("OWT-LowLatencyMode"))
+    return;
   bwe_incoming_ = bandwidth;
   CapBitrateToThresholds(now_ms, current_bitrate_bps_);
 }
@@ -410,11 +413,21 @@ void SendSideBandwidthEstimation::UpdateMinHistory(int64_t now_ms) {
 
 void SendSideBandwidthEstimation::CapBitrateToThresholds(int64_t now_ms,
                                                          uint32_t bitrate_bps) {
+  std::string experiment_string =
+      webrtc::field_trial::FindFullName("OWT-DelayBWEWeidght");
+  double delay_weight = ::strtod(experiment_string.c_str(), nullptr);
+  double delay_fraction = delay_weight / 100.0;
+  double lost_fraction = 1 - delay_fraction;
+
   if (bwe_incoming_ > 0 && bitrate_bps > bwe_incoming_) {
     bitrate_bps = bwe_incoming_;
   }
   if (delay_based_bitrate_bps_ > 0 && bitrate_bps > delay_based_bitrate_bps_) {
-    bitrate_bps = delay_based_bitrate_bps_;
+    if (delay_weight <= 100 && delay_weight >= 0)
+      bitrate_bps = bitrate_bps * lost_fraction +
+                    delay_based_bitrate_bps_ * delay_fraction;
+    else
+      bitrate_bps = delay_based_bitrate_bps_;
   }
   if (bitrate_bps > max_bitrate_configured_) {
     bitrate_bps = max_bitrate_configured_;

modules/congestion_controller/rtp/pacer_controller.cc

@@ -23,6 +23,9 @@ PacerController::~PacerController() = default;
 
 void PacerController::OnCongestionWindow(DataSize congestion_window) {
   RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_);
+  if (pacer_->IsLowLatencyMode()) {
+    pacer_->SetCongestionWindow(PacedSender::kNoCongestionWindow);
+  }
   if (congestion_window.IsFinite())
     pacer_->SetCongestionWindow(congestion_window.bytes());
   else

modules/pacing/paced_sender.cc

@@ -33,6 +33,7 @@
 namespace {
 // Time limit in milliseconds between packet bursts.
 const int64_t kMinPacketLimitMs = 5;
+const int64_t kMinPacketLimitLowLatency = 1; // For low latency mode we set process interval to 1ms instead.
 const int64_t kCongestedPacketIntervalMs = 500;
 const int64_t kPausedProcessIntervalMs = kCongestedPacketIntervalMs;
 const int64_t kMaxElapsedTimeMs = 2000;
@@ -64,6 +65,7 @@ PacedSender::PacedSender(const Clock* clock,
       packet_sender_(packet_sender),
       alr_detector_(absl::make_unique<AlrDetector>(event_log)),
       drain_large_queues_(!field_trial::IsDisabled("WebRTC-Pacer-DrainQueue")),
+      low_latency_mode_(field_trial::IsEnabled("OWT-LowLatencyMode")),
       send_padding_if_silent_(
           field_trial::IsEnabled("WebRTC-Pacer-PadInSilence")),
       video_blocks_audio_(!field_trial::IsDisabled("WebRTC-Pacer-BlockAudio")),
@@ -88,18 +90,27 @@ PacedSender::PacedSender(const Clock* clock,
     RTC_LOG(LS_WARNING) << "Pacer queues will not be drained,"
                            "pushback experiment must be enabled.";
   UpdateBudgetWithElapsedTime(kMinPacketLimitMs);
+  if (low_latency_mode_)
+    prober_->SetEnabled(false);
 }
 
 PacedSender::~PacedSender() {}
 
 void PacedSender::CreateProbeCluster(int bitrate_bps) {
   rtc::CritScope cs(&critsect_);
-  prober_->CreateProbeCluster(bitrate_bps, clock_->TimeInMilliseconds());
+  if (!low_latency_mode_) // Do not create probe cluster on low latency mode.
+    prober_->CreateProbeCluster(bitrate_bps, clock_->TimeInMilliseconds());
+}
+
+bool PacedSender::IsLowLatencyMode() const {
+  return low_latency_mode_;
 }
 
 void PacedSender::Pause() {
   {
     rtc::CritScope cs(&critsect_);
+    if (low_latency_mode_)
+      return;
     if (!paused_)
       RTC_LOG(LS_INFO) << "PacedSender paused.";
     paused_ = true;
@@ -115,6 +126,8 @@ void PacedSender::Pause() {
 void PacedSender::Resume() {
   {
     rtc::CritScope cs(&critsect_);
+    if (low_latency_mode_)
+      return;
     if (paused_)
       RTC_LOG(LS_INFO) << "PacedSender resumed.";
     paused_ = false;
@@ -146,6 +159,10 @@ bool PacedSender::Congested() const {
 void PacedSender::SetProbingEnabled(bool enabled) {
   rtc::CritScope cs(&critsect_);
   RTC_CHECK_EQ(0, packet_counter_);
+  if (IsLowLatencyMode()) {
+    // Always disable prober disabled
+    prober_->SetEnabled(false);
+  }
   prober_->SetEnabled(enabled);
 }
 
@@ -243,6 +260,8 @@ int64_t PacedSender::QueueInMs() const {
 
 int64_t PacedSender::TimeUntilNextProcess() {
   rtc::CritScope cs(&critsect_);
+  if (low_latency_mode_)
+    return kMinPacketLimitLowLatency;
   int64_t elapsed_time_us =
       clock_->TimeInMicroseconds() - time_last_process_us_;
   int64_t elapsed_time_ms = (elapsed_time_us + 500) / 1000;
@@ -260,6 +279,7 @@ int64_t PacedSender::TimeUntilNextProcess() {
 }
 
 void PacedSender::Process() {
+  RTC_LOG(LS_ERROR) << "p.";
   int64_t now_us = clock_->TimeInMicroseconds();
   rtc::CritScope cs(&critsect_);
   int64_t elapsed_time_ms = (now_us - time_last_process_us_ + 500) / 1000;
@@ -284,8 +304,10 @@ void PacedSender::Process() {
       }
     }
   }
-  if (paused_)
-    return;
+  if (paused_) {
+    RTC_LOG(LS_ERROR) << "Paused.";
+    //return;
+  }
 
   if (elapsed_time_ms > 0) {
     int target_bitrate_kbps = pacing_bitrate_kbps_;
@@ -319,7 +341,7 @@ void PacedSender::Process() {
   }
   // The paused state is checked in the loop since SendPacket leaves the
   // critical section allowing the paused state to be changed from other code.
-  while (!packets_->Empty() && !paused_) {
+  while (!packets_->Empty() /*&& !paused_*/) {
     // Since we need to release the lock in order to send, we first pop the
     // element from the priority queue but keep it in storage, so that we can
     // reinsert it if send fails.
@@ -329,31 +351,39 @@ void PacedSender::Process() {
       bytes_sent += packet.bytes;
       // Send succeeded, remove it from the queue.
       packets_->FinalizePop(packet);
+      RTC_LOG(LS_ERROR) << "PS#:" << packet.sequence_number
+                        << ",ca: " << packet.capture_time_ms
+                        << ",cu: " << clock_->TimeInMilliseconds()
+                        << ",sz: " << packet.bytes
+                        << ", q: " << packets_->SizeInPackets();
       if (is_probing && bytes_sent > recommended_probe_size)
         break;
     } else {
       // Send failed, put it back into the queue.
       packets_->CancelPop(packet);
+      RTC_LOG(LS_ERROR) << "f";
       break;
     }
   }
 
-  if (packets_->Empty() && !Congested()) {
-    // We can not send padding unless a normal packet has first been sent. If we
-    // do, timestamps get messed up.
-    if (packet_counter_ > 0) {
-      int padding_needed =
-          static_cast<int>(is_probing ? (recommended_probe_size - bytes_sent)
+  if (!IsLowLatencyMode()) {
+    if (packets_->Empty() && !Congested()) {
+      // We can not send padding unless a normal packet has first been sent. If we
+      // do, timestamps get messed up.
+      if (packet_counter_ > 0) {
+        int padding_needed =
+            static_cast<int>(is_probing ? (recommended_probe_size - bytes_sent)
                                       : padding_budget_->bytes_remaining());
-      if (padding_needed > 0) {
-        bytes_sent += SendPadding(padding_needed, pacing_info);
+        if (padding_needed > 0) {
+          bytes_sent += SendPadding(padding_needed, pacing_info);
+        }
       }
     }
-  }
-  if (is_probing) {
-    probing_send_failure_ = bytes_sent == 0;
-    if (!probing_send_failure_)
-      prober_->ProbeSent(clock_->TimeInMilliseconds(), bytes_sent);
+    if (is_probing) {
+      probing_send_failure_ = bytes_sent == 0;
+      if (!probing_send_failure_)
+        prober_->ProbeSent(clock_->TimeInMilliseconds(), bytes_sent);
+    }
   }
   alr_detector_->OnBytesSent(bytes_sent, now_us / 1000);
 }
@@ -370,18 +400,20 @@ bool PacedSender::SendPacket(const PacketQueueInterface::Packet& packet,
   bool audio_packet = packet.priority == kHighPriority;
   bool apply_pacing =
       !audio_packet || account_for_audio_ || video_blocks_audio_;
-  if (apply_pacing && (Congested() || (media_budget_->bytes_remaining() == 0 &&
+  if (!IsLowLatencyMode()) {
+    if (apply_pacing && (Congested() || (media_budget_->bytes_remaining() == 0 &&
                                        pacing_info.probe_cluster_id ==
                                            PacedPacketInfo::kNotAProbe))) {
-    return false;
+      return false;
+    }
   }
-
   critsect_.Leave();
+  RTC_LOG(LS_ERROR) << "b:" << clock_->TimeInMilliseconds();
   const bool success = packet_sender_->TimeToSendPacket(
       packet.ssrc, packet.sequence_number, packet.capture_time_ms,
       packet.retransmission, pacing_info);
   critsect_.Enter();
-
+  RTC_LOG(LS_ERROR) << "a:" << clock_->TimeInMilliseconds();
   if (success) {
     if (first_sent_packet_ms_ == -1)
       first_sent_packet_ms_ = clock_->TimeInMilliseconds();

modules/pacing/paced_sender.h

@@ -142,6 +142,8 @@ class PacedSender : public Pacer {
   void SetPacingFactor(float pacing_factor);
   void SetQueueTimeLimit(int limit_ms);
 
+  bool IsLowLatencyMode() const;
+
  private:
   // Updates the number of bytes that can be sent for the next time interval.
   void UpdateBudgetWithElapsedTime(int64_t delta_time_in_ms)
@@ -163,8 +165,11 @@ class PacedSender : public Pacer {
   const std::unique_ptr<AlrDetector> alr_detector_ RTC_PT_GUARDED_BY(critsect_);
 
   const bool drain_large_queues_;
+  const bool low_latency_mode_;
   const bool send_padding_if_silent_;
   const bool video_blocks_audio_;
+
+
   rtc::CriticalSection critsect_;
   bool paused_ RTC_GUARDED_BY(critsect_);
   // This is the media budget, keeping track of how many bits of media

modules/rtp_rtcp/include/rtp_rtcp_defines.h

@@ -107,6 +107,7 @@ enum RTPExtensionType {
   kRtpExtensionRepairedRtpStreamId,
   kRtpExtensionMid,
   kRtpExtensionGenericFrameDescriptor,
+  kRtpExtensionPictureId,  // Required by temporal scalability support
   kRtpExtensionNumberOfExtensions  // Must be the last entity in the enum.
 };