diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 1fbd1026f52..8800a030caa 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -324,7 +324,9 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( RTC_DCHECK_LT(stream_index, rtp_modules_.size()); RTPVideoHeader rtp_video_header = params_[stream_index].GetRtpVideoHeader( encoded_image, codec_specific_info, shared_frame_id_); - + if (codec_specific_info->codecSpecific.H264.last_fragment_in_frame) + absl::get(rtp_video_header.video_type_header) + .has_last_fragement = true; uint32_t frame_id; if (!rtp_modules_[stream_index]->Sending()) { // The payload router could be active but this module isn't sending. diff --git a/modules/rtp_rtcp/source/rtp_format.h b/modules/rtp_rtcp/source/rtp_format.h index 76d7f47ddbf..88728eb0d38 100644 --- a/modules/rtp_rtcp/source/rtp_format.h +++ b/modules/rtp_rtcp/source/rtp_format.h @@ -35,7 +35,7 @@ class RtpPacketizer { virtual size_t SetPayloadData( const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) = 0; + const RTPFragmentationHeader* fragmentation, bool end_of_frame) = 0; // Get the next payload with payload header. // Write payload and set marker bit of the |packet|. diff --git a/modules/rtp_rtcp/source/rtp_format_h264.cc b/modules/rtp_rtcp/source/rtp_format_h264.cc index 45e8d11fe2f..9c0a04f65cc 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -85,7 +85,9 @@ RtpPacketizerH264::RtpPacketizerH264(size_t max_payload_len, : max_payload_len_(max_payload_len), last_packet_reduction_len_(last_packet_reduction_len), num_packets_left_(0), - packetization_mode_(packetization_mode) { + packetization_mode_(packetization_mode), + end_of_frame_(true), + first_partial_fu_nal_header_(0) { // Guard against uninitialized memory in packetization_mode. RTC_CHECK(packetization_mode == H264PacketizationMode::NonInterleaved || packetization_mode == H264PacketizationMode::SingleNalUnit); @@ -96,26 +98,33 @@ RtpPacketizerH264::~RtpPacketizerH264() {} RtpPacketizerH264::Fragment::~Fragment() = default; -RtpPacketizerH264::Fragment::Fragment(const uint8_t* buffer, size_t length) - : buffer(buffer), length(length) {} +RtpPacketizerH264::Fragment::Fragment(const uint8_t* buffer, size_t length, uint8_t type) + : buffer(buffer), length(length), pl_type(type) {} RtpPacketizerH264::Fragment::Fragment(const Fragment& fragment) - : buffer(fragment.buffer), length(fragment.length) {} + : buffer(fragment.buffer), length(fragment.length), pl_type(fragment.pl_type) {} size_t RtpPacketizerH264::SetPayloadData( const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) { + const RTPFragmentationHeader* fragmentation, + bool end_of_frame) { RTC_DCHECK(packets_.empty()); RTC_DCHECK(input_fragments_.empty()); RTC_DCHECK(fragmentation); + end_of_frame_ = end_of_frame; + + // Different fragments allows to be packetized as FU or STAP-A or NAL, + // so there's no need to look back into history about how previous slice + // was packetized before. for (int i = 0; i < fragmentation->fragmentationVectorSize; ++i) { const uint8_t* buffer = &payload_data[fragmentation->fragmentationOffset[i]]; size_t length = fragmentation->fragmentationLength[i]; + uint8_t pl_type = fragmentation->fragmentationPlType[i]; bool updated_sps = false; H264::NaluType nalu_type = H264::ParseNaluType(buffer[0]); - if (nalu_type == H264::NaluType::kSps) { + if (pl_type == 0 && nalu_type == H264::NaluType::kSps) { // pl_type set to non-zero when current fragment is part of previous NAL. // Check if stream uses picture order count type 0, and if so rewrite it // to enable faster decoding. Streams in that format incur additional // delay because it allows decode order to differ from render order. @@ -141,7 +150,7 @@ size_t RtpPacketizerH264::SetPayloadData( switch (result) { case SpsVuiRewriter::ParseResult::kVuiRewritten: input_fragments_.push_back( - Fragment(output_buffer->data(), output_buffer->size())); + Fragment(output_buffer->data(), output_buffer->size(), pl_type)); input_fragments_.rbegin()->tmp_buffer = std::move(output_buffer); updated_sps = true; RTC_HISTOGRAM_ENUMERATION(kSpsValidHistogramName, @@ -167,7 +176,7 @@ size_t RtpPacketizerH264::SetPayloadData( } if (!updated_sps) - input_fragments_.push_back(Fragment(buffer, length)); + input_fragments_.push_back(Fragment(buffer, length, pl_type)); } if (!GeneratePackets()) { // If failed to generate all the packets, discard already generated @@ -183,6 +192,25 @@ size_t RtpPacketizerH264::SetPayloadData( } bool RtpPacketizerH264::GeneratePackets() { + // If there's only 1 fragement, handle specially. + if (input_fragments_.size() == 1 && input_fragments_[0].pl_type != 0) { + // It's part of previous NAL. S-bit must not be set. E-bit set if end_of_frame_ flag is set. + // BUGBUG: This is not neccessarily true, but for Titan SDK'case, there would be only 1-slice + // so OK to do so. + if (end_of_frame_) { + PacketizeFuA(0, true, false); + } else { + PacketizeFuA(0, false, false); + } + first_partial_fu_nal_header_ = input_fragments_[0].pl_type; + return true; + } + + // More than 1 fragments. + // cases: 1st part of previous NALU, last is not end of frame: FU first without S, with E; FU last with S, without E. + // 1st part of previous NALU, last is end of frame: FU first withou S, with E; Last with normal process. + // 1st is begin of frame, last is not end of frame: FU last with S and without E. + // 1st is begin of frame, last is end of fram. Normal process with all fragments. for (size_t i = 0; i < input_fragments_.size();) { switch (packetization_mode_) { case H264PacketizationMode::SingleNalUnit: @@ -192,13 +220,32 @@ bool RtpPacketizerH264::GeneratePackets() { break; case H264PacketizationMode::NonInterleaved: size_t fragment_len = input_fragments_[i].length; + + if (i == 0) { + if (input_fragments_[i].pl_type != 0) {// this is part of previous in-complete NAL + PacketizeFuA(0, true /*end*/, false /*begin*/); // S-bit must not be set. But E-bit depends + first_partial_fu_nal_header_ = input_fragments_[0].pl_type; + ++i; + break; + } + } if (i + 1 == input_fragments_.size()) { // Pretend that last fragment is larger instead of making last packet // smaller. fragment_len += last_packet_reduction_len_; + // If it's the last fragment in current payload and end_of_frame_ flag + // is not set, current packet must be fragmented into FU-A. + if (!end_of_frame_) { + PacketizeFuA(i, false, true); // If not the last slice of the frame, then E-bit must not be set. + } else { + PacketizeFuA(i, true, true); + } + ++i; + break; } + if (fragment_len > max_payload_len_) { - PacketizeFuA(i); + PacketizeFuA(i, true, true); ++i; } else { i = PacketizeStapA(i); @@ -209,13 +256,25 @@ bool RtpPacketizerH264::GeneratePackets() { return true; } -void RtpPacketizerH264::PacketizeFuA(size_t fragment_index) { +void RtpPacketizerH264::PacketizeFuA(size_t fragment_index, bool set_e_bit, bool set_s_bit) { // Fragment payload into packets (FU-A). // Strip out the original header and leave room for the FU-A header. const Fragment& fragment = input_fragments_[fragment_index]; bool is_last_fragment = fragment_index + 1 == input_fragments_.size(); + // override is_last_fragment flag + if (!set_e_bit) { + is_last_fragment = false; + } size_t payload_left = fragment.length - kNalHeaderSize; - size_t offset = kNalHeaderSize; + if (!set_s_bit) + payload_left = fragment.length; + + size_t offset; + if (set_s_bit) { + offset = kNalHeaderSize; + } else { + offset = 0; + } size_t per_packet_capacity = max_payload_len_ - kFuAHeaderSize; // Instead of making the last packet smaller we pretend that all packets are @@ -252,10 +311,11 @@ void RtpPacketizerH264::PacketizeFuA(size_t fragment_index) { } } RTC_CHECK_GT(packet_length, 0); - packets_.push(PacketUnit(Fragment(fragment.buffer + offset, packet_length), - offset - kNalHeaderSize == 0, - payload_left == packet_length, false, - fragment.buffer[0])); + // fragment.buffer[0] does not neccessarily contain the nal_header. So if set_s_bit is false, get the NAL header info from first input fragment header. + packets_.push(PacketUnit(Fragment(fragment.buffer + offset, packet_length, fragment.pl_type), + offset - kNalHeaderSize == 0, //if this FU is with offset to 0, it will also be marked as not first_fragment. + (payload_left == packet_length && set_e_bit), false, + set_s_bit? fragment.buffer[0] : first_partial_fu_nal_header_)); offset += packet_length; payload_left -= packet_length; --num_packets; @@ -348,7 +408,10 @@ bool RtpPacketizerH264::NextPacket(RtpPacketToSend* rtp_packet) { RTC_DCHECK_LE(rtp_packet->payload_size(), max_payload_len_ - last_packet_reduction_len_); } - rtp_packet->SetMarker(packets_.empty()); + // We will only set marker if this is really last slice + // of the AU. + if (end_of_frame_) + rtp_packet->SetMarker(packets_.empty()); --num_packets_left_; return true; } @@ -396,6 +459,7 @@ void RtpPacketizerH264::NextFragmentPacket(RtpPacketToSend* rtp_packet) { fu_header |= (packet->first_fragment ? kSBit : 0); fu_header |= (packet->last_fragment ? kEBit : 0); uint8_t type = packet->header & kTypeMask; + // fu_header |= type; const Fragment& fragment = packet->source_fragment; uint8_t* buffer = diff --git a/modules/rtp_rtcp/source/rtp_format_h264.h b/modules/rtp_rtcp/source/rtp_format_h264.h index 99b080b9ecc..dfdb7425989 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264.h +++ b/modules/rtp_rtcp/source/rtp_format_h264.h @@ -34,7 +34,8 @@ class RtpPacketizerH264 : public RtpPacketizer { size_t SetPayloadData(const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) override; + const RTPFragmentationHeader* fragmentation, + bool end_of_frame) override; // Get the next payload with H264 payload header. // Write payload and set marker bit of the |packet|. @@ -47,11 +48,12 @@ class RtpPacketizerH264 : public RtpPacketizer { // Input fragments (NAL units), with an optionally owned temporary buffer, // used in case the fragment gets modified. struct Fragment { - Fragment(const uint8_t* buffer, size_t length); + Fragment(const uint8_t* buffer, size_t length, uint8_t type); explicit Fragment(const Fragment& fragment); ~Fragment(); const uint8_t* buffer = nullptr; size_t length = 0; + uint8_t pl_type = 0; std::unique_ptr tmp_buffer; }; @@ -81,7 +83,9 @@ class RtpPacketizerH264 : public RtpPacketizer { }; bool GeneratePackets(); - void PacketizeFuA(size_t fragment_index); + // We modify the FU packetization to handle partial outputed NAL at the + // end or beginning of payload. + void PacketizeFuA(size_t fragment_index, bool slice_end, bool slice_begin); size_t PacketizeStapA(size_t fragment_index); bool PacketizeSingleNalu(size_t fragment_index); void NextAggregatePacket(RtpPacketToSend* rtp_packet, bool last); @@ -93,6 +97,8 @@ class RtpPacketizerH264 : public RtpPacketizer { const H264PacketizationMode packetization_mode_; std::deque input_fragments_; std::queue packets_; + bool end_of_frame_; + uint8_t first_partial_fu_nal_header_; RTC_DISALLOW_COPY_AND_ASSIGN(RtpPacketizerH264); }; diff --git a/modules/rtp_rtcp/source/rtp_format_h265.cc b/modules/rtp_rtcp/source/rtp_format_h265.cc index 8e8c48250aa..fd2117c9683 100644 --- a/modules/rtp_rtcp/source/rtp_format_h265.cc +++ b/modules/rtp_rtcp/source/rtp_format_h265.cc @@ -117,7 +117,8 @@ RtpPacketizerH265::Fragment::Fragment(const Fragment& fragment) size_t RtpPacketizerH265::SetPayloadData( const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) { + const RTPFragmentationHeader* fragmentation, + bool /*end_of_frame*/) { RTC_DCHECK(packets_.empty()); RTC_DCHECK(input_fragments_.empty()); RTC_DCHECK(fragmentation); diff --git a/modules/rtp_rtcp/source/rtp_format_h265.h b/modules/rtp_rtcp/source/rtp_format_h265.h index 6885fd29d73..37389db747f 100644 --- a/modules/rtp_rtcp/source/rtp_format_h265.h +++ b/modules/rtp_rtcp/source/rtp_format_h265.h @@ -25,7 +25,7 @@ class RtpPacketizerH265 : public RtpPacketizer { size_t SetPayloadData(const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) override; + const RTPFragmentationHeader* fragmentation, bool end_of_frame) override; // Get the next payload with H.265 payload header. // buffer is a pointer to where the output will be written. diff --git a/modules/rtp_rtcp/source/rtp_format_video_generic.cc b/modules/rtp_rtcp/source/rtp_format_video_generic.cc index 081a3bfa648..35bed7ceb85 100644 --- a/modules/rtp_rtcp/source/rtp_format_video_generic.cc +++ b/modules/rtp_rtcp/source/rtp_format_video_generic.cc @@ -43,7 +43,8 @@ RtpPacketizerGeneric::~RtpPacketizerGeneric() {} size_t RtpPacketizerGeneric::SetPayloadData( const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) { + const RTPFragmentationHeader* fragmentation, + bool /*end_of_frame*/) { payload_data_ = payload_data; payload_size_ = payload_size; diff --git a/modules/rtp_rtcp/source/rtp_format_video_generic.h b/modules/rtp_rtcp/source/rtp_format_video_generic.h index e608db59211..004467fe00f 100644 --- a/modules/rtp_rtcp/source/rtp_format_video_generic.h +++ b/modules/rtp_rtcp/source/rtp_format_video_generic.h @@ -39,7 +39,8 @@ class RtpPacketizerGeneric : public RtpPacketizer { // Returns total number of packets to be generated. size_t SetPayloadData(const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) override; + const RTPFragmentationHeader* fragmentation, + bool end_of_frame) override; // Get the next payload with generic payload header. // Write payload and set marker bit of the |packet|. diff --git a/modules/rtp_rtcp/source/rtp_format_vp8.cc b/modules/rtp_rtcp/source/rtp_format_vp8.cc index 48c73512abb..1c1ffa3ae34 100644 --- a/modules/rtp_rtcp/source/rtp_format_vp8.cc +++ b/modules/rtp_rtcp/source/rtp_format_vp8.cc @@ -175,7 +175,8 @@ RtpPacketizerVp8::~RtpPacketizerVp8() {} size_t RtpPacketizerVp8::SetPayloadData( const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* /* fragmentation */) { + const RTPFragmentationHeader* /* fragmentation */, + bool /*end_of_frame*/) { payload_data_ = payload_data; payload_size_ = payload_size; if (GeneratePackets() < 0) { diff --git a/modules/rtp_rtcp/source/rtp_format_vp8.h b/modules/rtp_rtcp/source/rtp_format_vp8.h index 2b4468abe48..d2cc09eee32 100644 --- a/modules/rtp_rtcp/source/rtp_format_vp8.h +++ b/modules/rtp_rtcp/source/rtp_format_vp8.h @@ -48,7 +48,8 @@ class RtpPacketizerVp8 : public RtpPacketizer { size_t SetPayloadData(const uint8_t* payload_data, size_t payload_size, - const RTPFragmentationHeader* fragmentation) override; + const RTPFragmentationHeader* fragmentation, + bool end_of_frame) override; // Get the next payload with VP8 payload header. // Write payload and set marker bit of the |packet|. diff --git a/modules/rtp_rtcp/source/rtp_format_vp9.cc b/modules/rtp_rtcp/source/rtp_format_vp9.cc index 974df8f5a9c..c83e01669ba 100644 --- a/modules/rtp_rtcp/source/rtp_format_vp9.cc +++ b/modules/rtp_rtcp/source/rtp_format_vp9.cc @@ -478,7 +478,8 @@ std::string RtpPacketizerVp9::ToString() { size_t RtpPacketizerVp9::SetPayloadData( const uint8_t* payload, size_t payload_size, - const RTPFragmentationHeader* fragmentation) { + const RTPFragmentationHeader* fragmentation, + bool /*end_of_frame*/) { payload_ = payload; payload_size_ = payload_size; GeneratePackets(); diff --git a/modules/rtp_rtcp/source/rtp_format_vp9.h b/modules/rtp_rtcp/source/rtp_format_vp9.h index 9052c21d678..626ee34b89d 100644 --- a/modules/rtp_rtcp/source/rtp_format_vp9.h +++ b/modules/rtp_rtcp/source/rtp_format_vp9.h @@ -43,7 +43,8 @@ class RtpPacketizerVp9 : public RtpPacketizer { // The payload data must be one encoded VP9 layer frame. size_t SetPayloadData(const uint8_t* payload, size_t payload_size, - const RTPFragmentationHeader* fragmentation) override; + const RTPFragmentationHeader* fragmentation, + bool end_of_frame) override; // Gets the next payload with VP9 payload header. // Write payload and set marker bit of the |packet|. diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index eca3a0dd944..e094a801621 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -295,6 +295,7 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type, size_t fec_packet_overhead; bool red_enabled; int32_t retransmission_settings; + bool frame_completed = true; { rtc::CritScope cs(&crit_); // According to @@ -307,7 +308,12 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type, // packet in each group of packets which make up another type of frame // (e.g. a P-Frame) only if the current value is different from the previous // value sent. - if (video_header) { + if (video_header && video_header->codec == kVideoCodecH264 && + !absl::get(video_header->video_type_header) + .has_last_fragement) { + frame_completed = false; + } + if (video_header && frame_completed) { // Set rotation when key frame or when changed (to follow standard). // Or when different from 0 (to follow current receiver implementation). VideoRotation current_rotation = video_header->rotation; @@ -363,8 +369,8 @@ bool RTPSenderVideo::SendVideo(enum VideoCodecType video_type, video_header ? GetTemporalId(*video_header) : kNoTemporalIdx; StorageType storage = GetStorageType(temporal_id, retransmission_settings, expected_retransmission_time_ms); - size_t num_packets = - packetizer->SetPayloadData(payload_data, payload_size, fragmentation); + size_t num_packets = packetizer->SetPayloadData( + payload_data, payload_size, fragmentation, frame_completed); if (num_packets == 0) return false; diff --git a/modules/video_coding/codecs/h264/include/h264_globals.h b/modules/video_coding/codecs/h264/include/h264_globals.h index 468f2720385..0f34a8cda01 100644 --- a/modules/video_coding/codecs/h264/include/h264_globals.h +++ b/modules/video_coding/codecs/h264/include/h264_globals.h @@ -81,6 +81,13 @@ struct RTPVideoHeaderH264 { // depending along with Temporal ID (obtained from RTP header extn). // '0' if PictureID does not exist. uint16_t picture_id; + // For support slice-based transmission, mark end of a frame so that + // the H.264 packetizer will not set marker bit for the last fragment of + // current outgoing data if it does not contain last fragment of the frame; + // and will treat the first fragment of the frame as continuous playload, so + // that it will not create FU header or STAP-A header on first fragment if contains + // last fragment of the frame. + bool has_last_fragement; }; } // namespace webrtc diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index 2cfc41e2c04..85fd5fab18b 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -71,6 +71,8 @@ struct CodecSpecificInfoH264 { H264PacketizationMode packetization_mode; uint8_t simulcast_idx; int16_t picture_id; // Required by temporal scalability + // Below is added to handle partial output from encoder. + bool last_fragment_in_frame; }; union CodecSpecificInfoUnion {