From 7bb6dd7e4010a5fa87c6948ad7536b4c29da956c Mon Sep 17 00:00:00 2001 From: Qiu Jianlin Date: Thu, 10 Oct 2019 11:18:48 +0800 Subject: [PATCH 1/2] Treat IDR_W_RADL and CRA as key frame as well. --- common_video/h265/h265_common.h | 2 +- modules/rtp_rtcp/source/rtp_format_h265.cc | 10 ++-- .../video_coding/h265_vps_sps_pps_tracker.cc | 6 ++- modules/video_coding/packet.cc | 2 +- modules/video_coding/packet_buffer.cc | 46 +++++++++++++++++-- 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/common_video/h265/h265_common.h b/common_video/h265/h265_common.h index 97acb42bb61..07bb2774ee1 100644 --- a/common_video/h265/h265_common.h +++ b/common_video/h265/h265_common.h @@ -43,7 +43,7 @@ enum NaluType : uint8_t { kBlaWRadl = 17, kBlaNLp = 18, kIdrWRadl = 19, - kIdr = 20, + kIdrNLp = 20, kCra = 21, kRsvIrapVcl23 = 23, kVps = 32, diff --git a/modules/rtp_rtcp/source/rtp_format_h265.cc b/modules/rtp_rtcp/source/rtp_format_h265.cc index d5b304e4334..88b40f9b619 100644 --- a/modules/rtp_rtcp/source/rtp_format_h265.cc +++ b/modules/rtp_rtcp/source/rtp_format_h265.cc @@ -482,7 +482,6 @@ bool RtpDepacketizerH265::ProcessApOrSingleNalu( nalu.sps_id = -1; nalu.pps_id = -1; start_offset += kHevcNalHeaderSize; - switch (nalu.type) { case H265::NaluType::kVps: { absl::optional vps = H265VpsParser::ParseVps( @@ -532,7 +531,9 @@ bool RtpDepacketizerH265::ProcessApOrSingleNalu( } break; } - case H265::NaluType::kIdr: + case H265::NaluType::kIdrWRadl: + case H265::NaluType::kIdrNLp: + case H265::NaluType::kCra: parsed_payload->video_header().frame_type = VideoFrameType::kVideoFrameKey; RTC_FALLTHROUGH(); case H265::NaluType::kTrailN: @@ -559,7 +560,6 @@ bool RtpDepacketizerH265::ProcessApOrSingleNalu( case H265::NaluType::kRadlR: case H265::NaluType::kBlaWLp: case H265::NaluType::kBlaWRadl: - case H265::NaluType::kIdrWRadl: case H265::NaluType::kPrefixSei: case H265::NaluType::kSuffixSei: break; @@ -622,7 +622,9 @@ bool RtpDepacketizerH265::ParseFuNalu( length_ -= (kHevcNalHeaderSize + kHevcFuHeaderSize); } - if (original_nal_type == H265::NaluType::kIdr) { + if (original_nal_type == H265::NaluType::kIdrWRadl + || original_nal_type == H265::NaluType::kIdrNLp + || original_nal_type == H265::NaluType::kCra) { parsed_payload->video_header().frame_type = VideoFrameType::kVideoFrameKey; } else { parsed_payload->video_header().frame_type = VideoFrameType::kVideoFrameDelta; diff --git a/modules/video_coding/h265_vps_sps_pps_tracker.cc b/modules/video_coding/h265_vps_sps_pps_tracker.cc index 088feac3412..9c38373d9f7 100644 --- a/modules/video_coding/h265_vps_sps_pps_tracker.cc +++ b/modules/video_coding/h265_vps_sps_pps_tracker.cc @@ -64,7 +64,9 @@ H265VpsSpsPpsTracker::PacketAction H265VpsSpsPpsTracker::CopyAndFixBitstream( pps_data_[nalu.pps_id].sps_id = nalu.sps_id; break; } - case H265::NaluType::kIdr: { + case H265::NaluType::kIdrWRadl: + case H265::NaluType::kIdrNLp: + case H265::NaluType::kCra: { // If this is the first packet of an IDR, make sure we have the required // SPS/PPS and also calculate how much extra space we need in the buffer // to prepend the SPS/PPS to the bitstream with start codes. @@ -193,7 +195,7 @@ H265VpsSpsPpsTracker::PacketAction H265VpsSpsPpsTracker::CopyAndFixBitstream( h265_header.nalus[h265_header.nalus_length++] = sps_info; h265_header.nalus[h265_header.nalus_length++] = pps_info; } else { - RTC_LOG(LS_WARNING) << "Not enough space in H.264 codec header to insert " + RTC_LOG(LS_WARNING) << "Not enough space in H.265 codec header to insert " "SPS/PPS provided out-of-band."; } } diff --git a/modules/video_coding/packet.cc b/modules/video_coding/packet.cc index 46df82ac64a..9b0bf7c9d79 100644 --- a/modules/video_coding/packet.cc +++ b/modules/video_coding/packet.cc @@ -44,7 +44,7 @@ VCMPacket::VCMPacket(const uint8_t* ptr, markerBit(rtp_header.markerBit), timesNacked(-1), completeNALU(kNaluIncomplete), - insertStartCode(videoHeader.codec == kVideoCodecH264 && + insertStartCode((videoHeader.codec == kVideoCodecH264 || videoHeader.codec == kVideoCodecH265) && videoHeader.is_first_packet_in_frame), video_header(videoHeader) { if (is_first_packet_in_frame() && markerBit) { diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index 22d9de96fad..d8502dc7c46 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -326,7 +326,7 @@ std::vector> PacketBuffer::FindFrames( max_recv_time = std::max(max_recv_time, data_buffer_[start_index].receive_time_ms); - if (!is_h264 && sequence_buffer_[start_index].frame_begin) + if (!is_h264 && !is_h265 && sequence_buffer_[start_index].frame_begin) break; if (is_h264 && !is_h264_keyframe) { @@ -361,7 +361,9 @@ std::vector> PacketBuffer::FindFrames( has_h265_sps = true; } else if (h265_header->nalus[j].type == H265::NaluType::kPps) { has_h265_pps = true; - } else if (h265_header->nalus[j].type == H265::NaluType::kIdr) { + } else if (h265_header->nalus[j].type == H265::NaluType::kIdrWRadl + || h265_header->nalus[j].type == H265::NaluType::kIdrNLp + || h265_header->nalus[j].type == H265::NaluType::kCra) { has_h265_idr = true; } } @@ -381,7 +383,7 @@ std::vector> PacketBuffer::FindFrames( // the timestamp of that packet is the same as this one. This may cause // the PacketBuffer to hand out incomplete frames. // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7106 - if (is_h264 && + if ((is_h264 || is_h265) && (!sequence_buffer_[start_index].used || data_buffer_[start_index].timestamp != frame_timestamp)) { break; @@ -432,6 +434,44 @@ std::vector> PacketBuffer::FindFrames( } } +#ifndef DISABLE_H265 + if (is_h265) { + // Warn if this is an unsafe frame. + if (has_h265_idr && (!has_h265_sps || !has_h265_pps)) { + std::stringstream ss; + ss << "Received H.265-IDR frame " + << "(SPS: " << has_h265_sps << ", PPS: " << has_h265_pps << "). "; + ss << "Treating as delta frame since " + "WebRTC-SpsPpsIdrIsH265Keyframe is always enabled."; + RTC_LOG(LS_WARNING) << ss.str(); + } + + // Now that we have decided whether to treat this frame as a key frame + // or delta frame in the frame buffer, we update the field that + // determines if the RtpFrameObject is a key frame or delta frame. + const size_t first_packet_index = start_seq_num % size_; + RTC_CHECK_LT(first_packet_index, size_); + if (is_h265_keyframe) { + data_buffer_[first_packet_index].video_header.frame_type = VideoFrameType::kVideoFrameKey; + } else { + data_buffer_[first_packet_index].video_header.frame_type = VideoFrameType::kVideoFrameDelta; + } + + // If this is not a key frame, make sure there are no gaps in the + // packet sequence numbers up until this point. + if (!is_h265_keyframe && missing_packets_.upper_bound(start_seq_num) != + missing_packets_.begin()) { + uint16_t stop_index = (index + 1) % size_; + while (start_index != stop_index) { + sequence_buffer_[start_index].frame_created = false; + start_index = (start_index + 1) % size_; + } + + return found_frames; + } + } +#endif + missing_packets_.erase(missing_packets_.begin(), missing_packets_.upper_bound(seq_num)); From f3ad3b51fdb0622e488b69b6ccc215a4fe7eb6d4 Mon Sep 17 00:00:00 2001 From: Qiu Jianlin Date: Fri, 11 Oct 2019 08:32:32 +0800 Subject: [PATCH 2/2] Fix h.265 subscription and publication issues --- call/rtp_payload_params.cc | 8 ++++++++ modules/rtp_rtcp/source/rtp_format_h265.cc | 5 +++-- modules/video_coding/include/video_codec_interface.h | 11 +++++++++++ modules/video_coding/packet.cc | 5 +++++ modules/video_coding/packet_buffer.cc | 4 +++- 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index 4b80e2c8d2e..431fd55145e 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -101,6 +101,14 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, } return; } +#ifndef DISABLE_H265 + case kVideoCodecH265: { + auto h265_header = rtp->video_type_header.emplace(); + h265_header.packetization_mode = + info.codecSpecific.H265.packetization_mode; + } + return; +#endif case kVideoCodecMultiplex: case kVideoCodecGeneric: rtp->codec = kVideoCodecGeneric; diff --git a/modules/rtp_rtcp/source/rtp_format_h265.cc b/modules/rtp_rtcp/source/rtp_format_h265.cc index 88b40f9b619..8026be48b91 100644 --- a/modules/rtp_rtcp/source/rtp_format_h265.cc +++ b/modules/rtp_rtcp/source/rtp_format_h265.cc @@ -171,7 +171,7 @@ bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) { // Strip out the original header and leave room for the FU header. const Fragment& fragment = input_fragments_[fragment_index]; PayloadSizeLimits limits = limits_; - limits.max_payload_len -= kHevcFuHeaderSize; + limits.max_payload_len -= kHevcFuHeaderSize + kHevcNalHeaderSize; // Update single/first/last packet reductions unless it is single/first/last // fragment. @@ -202,10 +202,11 @@ bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) { for (size_t i = 0; i < payload_sizes.size(); ++i) { int packet_length = payload_sizes[i]; RTC_CHECK_GT(packet_length, 0); + uint16_t header = (fragment.buffer[0] << 8) | fragment.buffer[1]; packets_.push(PacketUnit(Fragment(fragment.buffer + offset, packet_length), /*first_fragment=*/i == 0, /*last_fragment=*/i == payload_sizes.size() - 1, - false, fragment.buffer[0])); + false, header)); offset += packet_length; payload_left -= packet_length; } diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index 4aa13ac9648..04d07d0cb08 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -87,12 +87,23 @@ struct CodecSpecificInfoH264 { bool base_layer_sync; bool idr_frame; }; + +#ifndef DISABLE_H265 +struct CodecSpecificInfoH265 { + H265PacketizationMode packetization_mode; + bool idr_frame; +}; +#endif + static_assert(std::is_pod::value, ""); union CodecSpecificInfoUnion { CodecSpecificInfoVP8 VP8; CodecSpecificInfoVP9 VP9; CodecSpecificInfoH264 H264; +#ifndef DISABLE_H265 + CodecSpecificInfoH265 H265; +#endif }; static_assert(std::is_pod::value, ""); diff --git a/modules/video_coding/packet.cc b/modules/video_coding/packet.cc index 9b0bf7c9d79..176bb49b0c2 100644 --- a/modules/video_coding/packet.cc +++ b/modules/video_coding/packet.cc @@ -44,8 +44,13 @@ VCMPacket::VCMPacket(const uint8_t* ptr, markerBit(rtp_header.markerBit), timesNacked(-1), completeNALU(kNaluIncomplete), +#ifndef DISABLE_H265 insertStartCode((videoHeader.codec == kVideoCodecH264 || videoHeader.codec == kVideoCodecH265) && videoHeader.is_first_packet_in_frame), +#else + insertStartCode(videoHeader.codec == kVideoCodecH264 && + videoHeader.is_first_packet_in_frame), +#endif video_header(videoHeader) { if (is_first_packet_in_frame() && markerBit) { completeNALU = kNaluComplete; diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index d8502dc7c46..779cffcb87a 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -307,8 +307,10 @@ std::vector> PacketBuffer::FindFrames( bool has_h264_pps = false; bool has_h264_idr = false; bool is_h264_keyframe = false; + + bool is_h265 = false; #ifndef DISABLE_H265 - bool is_h265 = data_buffer_[start_index].codec() == kVideoCodecH265; + is_h265 = data_buffer_[start_index].codec() == kVideoCodecH265; bool has_h265_sps = false; bool has_h265_pps = false; bool has_h265_idr = false;