WIP: [Do Not Merge]Enable partial-output packetization for slice-based encoding

call/rtp_video_sender.cc

@@ -324,7 +324,9 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage(
   RTC_DCHECK_LT(stream_index, rtp_modules_.size());
   RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader(
       encoded_image, codec_specific_info, shared_frame_id_);
-
+  if (codec_specific_info->codecSpecific.H264.last_fragment_in_frame)
+    absl::get<RTPVideoHeaderH264>(rtp_video_header.video_type_header)
+        .has_last_fragement = true;
   uint32_t frame_id;
   if (!rtp_modules_[stream_index]->Sending()) {
     // The payload router could be active but this module isn't sending.

modules/rtp_rtcp/source/rtp_format.h

@@ -35,7 +35,7 @@ class RtpPacketizer {
   virtual size_t SetPayloadData(
       const uint8_t* payload_data,
       size_t payload_size,
-      const RTPFragmentationHeader* fragmentation) = 0;
+      const RTPFragmentationHeader* fragmentation, bool end_of_frame) = 0;
 
   // Get the next payload with payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_format_h264.cc

@@ -85,7 +85,9 @@ RtpPacketizerH264::RtpPacketizerH264(size_t max_payload_len,
     : max_payload_len_(max_payload_len),
       last_packet_reduction_len_(last_packet_reduction_len),
       num_packets_left_(0),
-      packetization_mode_(packetization_mode) {
+      packetization_mode_(packetization_mode),
+      end_of_frame_(true),
+      first_partial_fu_nal_header_(0) {
   // Guard against uninitialized memory in packetization_mode.
   RTC_CHECK(packetization_mode == H264PacketizationMode::NonInterleaved ||
             packetization_mode == H264PacketizationMode::SingleNalUnit);
@@ -96,26 +98,33 @@ RtpPacketizerH264::~RtpPacketizerH264() {}
 
 RtpPacketizerH264::Fragment::~Fragment() = default;
 
-RtpPacketizerH264::Fragment::Fragment(const uint8_t* buffer, size_t length)
-    : buffer(buffer), length(length) {}
+RtpPacketizerH264::Fragment::Fragment(const uint8_t* buffer, size_t length, uint8_t type)
+    : buffer(buffer), length(length), pl_type(type) {}
 RtpPacketizerH264::Fragment::Fragment(const Fragment& fragment)
-    : buffer(fragment.buffer), length(fragment.length) {}
+    : buffer(fragment.buffer), length(fragment.length), pl_type(fragment.pl_type) {}
 
 size_t RtpPacketizerH264::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool end_of_frame) {
   RTC_DCHECK(packets_.empty());
   RTC_DCHECK(input_fragments_.empty());
   RTC_DCHECK(fragmentation);
+  end_of_frame_ = end_of_frame;
+
+  // Different fragments allows to be packetized as FU or STAP-A or NAL,
+  // so there's no need to look back into history about how previous slice
+  // was packetized before.
   for (int i = 0; i < fragmentation->fragmentationVectorSize; ++i) {
     const uint8_t* buffer =
         &payload_data[fragmentation->fragmentationOffset[i]];
     size_t length = fragmentation->fragmentationLength[i];
+    uint8_t pl_type = fragmentation->fragmentationPlType[i];
 
     bool updated_sps = false;
     H264::NaluType nalu_type = H264::ParseNaluType(buffer[0]);
-    if (nalu_type == H264::NaluType::kSps) {
+    if (pl_type == 0 && nalu_type == H264::NaluType::kSps) {  // pl_type set to non-zero when current fragment is part of previous NAL.
       // Check if stream uses picture order count type 0, and if so rewrite it
       // to enable faster decoding. Streams in that format incur additional
       // delay because it allows decode order to differ from render order.
@@ -141,7 +150,7 @@ size_t RtpPacketizerH264::SetPayloadData(
       switch (result) {
         case SpsVuiRewriter::ParseResult::kVuiRewritten:
           input_fragments_.push_back(
-              Fragment(output_buffer->data(), output_buffer->size()));
+              Fragment(output_buffer->data(), output_buffer->size(), pl_type));
           input_fragments_.rbegin()->tmp_buffer = std::move(output_buffer);
           updated_sps = true;
           RTC_HISTOGRAM_ENUMERATION(kSpsValidHistogramName,
@@ -167,7 +176,7 @@ size_t RtpPacketizerH264::SetPayloadData(
     }
 
     if (!updated_sps)
-      input_fragments_.push_back(Fragment(buffer, length));
+      input_fragments_.push_back(Fragment(buffer, length, pl_type));
   }
   if (!GeneratePackets()) {
     // If failed to generate all the packets, discard already generated
@@ -183,6 +192,25 @@ size_t RtpPacketizerH264::SetPayloadData(
 }
 
 bool RtpPacketizerH264::GeneratePackets() {
+  // If there's only 1 fragement, handle specially.
+  if (input_fragments_.size() == 1 && input_fragments_[0].pl_type != 0) {
+    // It's part of previous NAL. S-bit must not be set. E-bit set if end_of_frame_ flag is set.
+    // BUGBUG: This is not neccessarily true, but for Titan SDK'case, there would be only 1-slice
+    // so OK to do so.
+    if (end_of_frame_) {
+      PacketizeFuA(0, true, false);
+    } else {
+      PacketizeFuA(0, false, false);
+    }
+    first_partial_fu_nal_header_ = input_fragments_[0].pl_type;
+    return true;
+  }
+
+  // More than 1 fragments.
+  // cases:   1st part of previous NALU, last is not end of frame: FU first without S, with E;  FU last with S, without E.
+  //          1st part of previous NALU,  last is end of frame:  FU first withou S, with E;  Last with normal process.
+  //          1st is begin of frame, last is not end of frame:  FU last with S and without E.
+  //          1st is begin of frame, last is end of fram. Normal process with all fragments.
   for (size_t i = 0; i < input_fragments_.size();) {
     switch (packetization_mode_) {
       case H264PacketizationMode::SingleNalUnit:
@@ -192,13 +220,32 @@ bool RtpPacketizerH264::GeneratePackets() {
         break;
       case H264PacketizationMode::NonInterleaved:
         size_t fragment_len = input_fragments_[i].length;
+
+        if (i == 0) {
+          if (input_fragments_[i].pl_type != 0) {// this is part of previous in-complete NAL
+            PacketizeFuA(0, true /*end*/, false /*begin*/); // S-bit must not be set. But E-bit depends
+            first_partial_fu_nal_header_ = input_fragments_[0].pl_type;
+            ++i;
+            break;
+          }
+        }
         if (i + 1 == input_fragments_.size()) {
           // Pretend that last fragment is larger instead of making last packet
           // smaller.
           fragment_len += last_packet_reduction_len_;
+          // If it's the last fragment in current payload and end_of_frame_ flag
+          // is not set, current packet must be fragmented into FU-A.
+          if (!end_of_frame_) {
+            PacketizeFuA(i, false, true);    // If not the last slice of the frame, then E-bit must not be set.
+          } else {
+            PacketizeFuA(i, true, true);
+          }
+          ++i;
+          break;
         }
+
         if (fragment_len > max_payload_len_) {
-          PacketizeFuA(i);
+          PacketizeFuA(i, true, true);
           ++i;
         } else {
           i = PacketizeStapA(i);
@@ -209,13 +256,25 @@ bool RtpPacketizerH264::GeneratePackets() {
   return true;
 }
 
-void RtpPacketizerH264::PacketizeFuA(size_t fragment_index) {
+void RtpPacketizerH264::PacketizeFuA(size_t fragment_index, bool set_e_bit, bool set_s_bit) {
   // Fragment payload into packets (FU-A).
   // Strip out the original header and leave room for the FU-A header.
   const Fragment& fragment = input_fragments_[fragment_index];
   bool is_last_fragment = fragment_index + 1 == input_fragments_.size();
+  // override is_last_fragment flag
+  if (!set_e_bit) {
+    is_last_fragment = false;
+  }
   size_t payload_left = fragment.length - kNalHeaderSize;
-  size_t offset = kNalHeaderSize;
+  if (!set_s_bit)
+    payload_left = fragment.length;
+
+  size_t offset;
+  if (set_s_bit) {
+    offset = kNalHeaderSize;
+  } else {
+    offset = 0;
+  }
   size_t per_packet_capacity = max_payload_len_ - kFuAHeaderSize;
 
   // Instead of making the last packet smaller we pretend that all packets are
@@ -252,10 +311,11 @@ void RtpPacketizerH264::PacketizeFuA(size_t fragment_index) {
       }
     }
     RTC_CHECK_GT(packet_length, 0);
-    packets_.push(PacketUnit(Fragment(fragment.buffer + offset, packet_length),
-                             offset - kNalHeaderSize == 0,
-                             payload_left == packet_length, false,
-                             fragment.buffer[0]));
+    // fragment.buffer[0] does not neccessarily contain the nal_header. So if set_s_bit is false, get the NAL header info from first input fragment header.
+    packets_.push(PacketUnit(Fragment(fragment.buffer + offset, packet_length, fragment.pl_type),
+                             offset - kNalHeaderSize == 0,  //if this FU is with offset to 0, it will also be marked as not first_fragment.
+                             (payload_left == packet_length && set_e_bit), false,
+                             set_s_bit? fragment.buffer[0] : first_partial_fu_nal_header_));
     offset += packet_length;
     payload_left -= packet_length;
     --num_packets;
@@ -348,7 +408,10 @@ bool RtpPacketizerH264::NextPacket(RtpPacketToSend* rtp_packet) {
     RTC_DCHECK_LE(rtp_packet->payload_size(),
                   max_payload_len_ - last_packet_reduction_len_);
   }
-  rtp_packet->SetMarker(packets_.empty());
+  // We will only set marker if this is really last slice
+  // of the AU.
+  if (end_of_frame_)
+    rtp_packet->SetMarker(packets_.empty());
   --num_packets_left_;
   return true;
 }
@@ -396,6 +459,7 @@ void RtpPacketizerH264::NextFragmentPacket(RtpPacketToSend* rtp_packet) {
   fu_header |= (packet->first_fragment ? kSBit : 0);
   fu_header |= (packet->last_fragment ? kEBit : 0);
   uint8_t type = packet->header & kTypeMask;
+  //
   fu_header |= type;
   const Fragment& fragment = packet->source_fragment;
   uint8_t* buffer =

modules/rtp_rtcp/source/rtp_format_h264.h

@@ -34,7 +34,8 @@ class RtpPacketizerH264 : public RtpPacketizer {
 
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Get the next payload with H264 payload header.
   // Write payload and set marker bit of the |packet|.
@@ -47,11 +48,12 @@ class RtpPacketizerH264 : public RtpPacketizer {
   // Input fragments (NAL units), with an optionally owned temporary buffer,
   // used in case the fragment gets modified.
   struct Fragment {
-    Fragment(const uint8_t* buffer, size_t length);
+    Fragment(const uint8_t* buffer, size_t length, uint8_t type);
     explicit Fragment(const Fragment& fragment);
     ~Fragment();
     const uint8_t* buffer = nullptr;
     size_t length = 0;
+    uint8_t pl_type = 0;
     std::unique_ptr<rtc::Buffer> tmp_buffer;
   };
 
@@ -81,7 +83,9 @@ class RtpPacketizerH264 : public RtpPacketizer {
   };
 
   bool GeneratePackets();
-  void PacketizeFuA(size_t fragment_index);
+  // We modify the FU packetization to handle partial outputed NAL at the
+  // end or beginning of payload.
+  void PacketizeFuA(size_t fragment_index, bool slice_end, bool slice_begin);
   size_t PacketizeStapA(size_t fragment_index);
   bool PacketizeSingleNalu(size_t fragment_index);
   void NextAggregatePacket(RtpPacketToSend* rtp_packet, bool last);
@@ -93,6 +97,8 @@ class RtpPacketizerH264 : public RtpPacketizer {
   const H264PacketizationMode packetization_mode_;
   std::deque<Fragment> input_fragments_;
   std::queue<PacketUnit> packets_;
+  bool end_of_frame_;
+  uint8_t first_partial_fu_nal_header_;
 
   RTC_DISALLOW_COPY_AND_ASSIGN(RtpPacketizerH264);
 };

modules/rtp_rtcp/source/rtp_format_h265.cc

@@ -117,7 +117,8 @@ RtpPacketizerH265::Fragment::Fragment(const Fragment& fragment)
 size_t RtpPacketizerH265::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool /*end_of_frame*/) {
   RTC_DCHECK(packets_.empty());
   RTC_DCHECK(input_fragments_.empty());
   RTC_DCHECK(fragmentation);

modules/rtp_rtcp/source/rtp_format_h265.h

@@ -25,7 +25,7 @@ class RtpPacketizerH265 : public RtpPacketizer {
 
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation, bool end_of_frame) override;
 
   // Get the next payload with H.265 payload header.
   // buffer is a pointer to where the output will be written.

modules/rtp_rtcp/source/rtp_format_video_generic.cc

@@ -43,7 +43,8 @@ RtpPacketizerGeneric::~RtpPacketizerGeneric() {}
 size_t RtpPacketizerGeneric::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool /*end_of_frame*/) {
   payload_data_ = payload_data;
   payload_size_ = payload_size;
 

modules/rtp_rtcp/source/rtp_format_video_generic.h

@@ -39,7 +39,8 @@ class RtpPacketizerGeneric : public RtpPacketizer {
   // Returns total number of packets to be generated.
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Get the next payload with generic payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_format_vp8.cc

@@ -175,7 +175,8 @@ RtpPacketizerVp8::~RtpPacketizerVp8() {}
 size_t RtpPacketizerVp8::SetPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
-    const RTPFragmentationHeader* /* fragmentation */) {
+    const RTPFragmentationHeader* /* fragmentation */,
+    bool /*end_of_frame*/) {
   payload_data_ = payload_data;
   payload_size_ = payload_size;
   if (GeneratePackets() < 0) {

modules/rtp_rtcp/source/rtp_format_vp8.h

@@ -48,7 +48,8 @@ class RtpPacketizerVp8 : public RtpPacketizer {
 
   size_t SetPayloadData(const uint8_t* payload_data,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Get the next payload with VP8 payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_format_vp9.cc

@@ -478,7 +478,8 @@ std::string RtpPacketizerVp9::ToString() {
 size_t RtpPacketizerVp9::SetPayloadData(
     const uint8_t* payload,
     size_t payload_size,
-    const RTPFragmentationHeader* fragmentation) {
+    const RTPFragmentationHeader* fragmentation,
+    bool /*end_of_frame*/) {
   payload_ = payload;
   payload_size_ = payload_size;
   GeneratePackets();

modules/rtp_rtcp/source/rtp_format_vp9.h

@@ -43,7 +43,8 @@ class RtpPacketizerVp9 : public RtpPacketizer {
   // The payload data must be one encoded VP9 layer frame.
   size_t SetPayloadData(const uint8_t* payload,
                         size_t payload_size,
-                        const RTPFragmentationHeader* fragmentation) override;
+                        const RTPFragmentationHeader* fragmentation,
+                        bool end_of_frame) override;
 
   // Gets the next payload with VP9 payload header.
   // Write payload and set marker bit of the |packet|.

modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -295,6 +295,7 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type,
   size_t fec_packet_overhead;
   bool red_enabled;
   int32_t retransmission_settings;
+  bool frame_completed = true;
   {
     rtc::CritScope cs(&crit_);
     // According to
@@ -307,7 +308,12 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type,
     // packet in each group of packets which make up another type of frame
     // (e.g. a P-Frame) only if the current value is different from the previous
     // value sent.
-    if (video_header) {
+    if (video_header && video_header->codec == kVideoCodecH264 &&
+        !absl::get<RTPVideoHeaderH264>(video_header->video_type_header)
+             .has_last_fragement) {
+      frame_completed = false;
+    }
+    if (video_header && frame_completed) {
       // Set rotation when key frame or when changed (to follow standard).
       // Or when different from 0 (to follow current receiver implementation).
       VideoRotation current_rotation = video_header->rotation;
@@ -363,8 +369,8 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type,
       video_header ? GetTemporalId(*video_header) : kNoTemporalIdx;
   StorageType storage = GetStorageType(temporal_id, retransmission_settings,
                                        expected_retransmission_time_ms);
-  size_t num_packets =
-      packetizer->SetPayloadData(payload_data, payload_size, fragmentation);
+  size_t num_packets = packetizer->SetPayloadData(
+      payload_data, payload_size, fragmentation, frame_completed);
 
   if (num_packets == 0)
     return false;

modules/video_coding/codecs/h264/include/h264_globals.h

@@ -81,6 +81,13 @@ struct RTPVideoHeaderH264 {
   // depending along with Temporal ID (obtained from RTP header extn).
   // '0' if PictureID does not exist.
   uint16_t picture_id;
+  // For support slice-based transmission, mark end of a frame so that
+  // the H.264 packetizer will not set marker bit for the last fragment of
+  // current outgoing data if it does not contain last fragment of the frame;
+  // and will treat the first fragment of the frame as continuous playload, so
+  // that it will not create FU header or STAP-A header on first fragment if contains
+  // last fragment of the frame.
+  bool has_last_fragement;
 };
 
 }  // namespace webrtc

modules/video_coding/include/video_codec_interface.h

@@ -71,6 +71,8 @@ struct CodecSpecificInfoH264 {
   H264PacketizationMode packetization_mode;
   uint8_t simulcast_idx;
   int16_t picture_id; // Required by temporal scalability
+  // Below is added to handle partial output from encoder.
+  bool last_fragment_in_frame;
 };
 
 union CodecSpecificInfoUnion {