From ab4ba6a09f55be66d60d61d1a71c62f310b318d2 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 23 Nov 2022 11:16:41 +0800 Subject: [PATCH] H265: Support parse multiple NALUs in a frame. v6.0.3 1. Fix parsing multiple NALUs bug. 2. Eliminate duplicated code for parsing NALU. 3. Return error when HEVC not enabled. --- trunk/doc/CHANGELOG.md | 1 + trunk/ide/srs_clion/CMakeLists.txt | 2 +- trunk/src/core/srs_core_version6.hpp | 2 +- trunk/src/kernel/srs_kernel_codec.cpp | 117 +++++++++----------------- trunk/src/kernel/srs_kernel_codec.hpp | 1 - trunk/src/utest/srs_utest_kernel.cpp | 19 +++++ 6 files changed, 61 insertions(+), 81 deletions(-) diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index e29b973a89..bafd482123 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -8,6 +8,7 @@ The changelog for SRS. ## SRS 6.0 Changelog +* v6.0, 2022-11-23, Merge [#3274](https://github.com/ossrs/srs/pull/3274): H265: Support parse multiple NALUs in a frame. v6.0.3 * v6.0, 2022-11-22, Merge [#3272](https://github.com/ossrs/srs/pull/3272): H265: Support HEVC over RTMP or HTTP-FLV. v6.0.2 * v6.0, 2022-11-22, Merge [#3268](https://github.com/ossrs/srs/pull/3268): H265: Update mpegts.js to play HEVC over HTTP-TS/FLV. v6.0.1 * v6.0, 2022-11-22, Init SRS 6. v6.0.0 diff --git a/trunk/ide/srs_clion/CMakeLists.txt b/trunk/ide/srs_clion/CMakeLists.txt index 1bfbcf25dd..ce4594cb56 100755 --- a/trunk/ide/srs_clion/CMakeLists.txt +++ b/trunk/ide/srs_clion/CMakeLists.txt @@ -31,7 +31,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") WORKING_DIRECTORY ${SRS_DIR} RESULT_VARIABLE ret) ELSE () EXECUTE_PROCESS( - COMMAND ./configure --srt=on --gb28181=on --utest=on --jobs=${JOBS} + COMMAND ./configure --srt=on --gb28181=on --h265=on --utest=on --jobs=${JOBS} WORKING_DIRECTORY ${SRS_DIR} RESULT_VARIABLE ret) ENDIF () if(NOT ret EQUAL 0) diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp index 65873f752f..e18455ca33 100644 --- a/trunk/src/core/srs_core_version6.hpp +++ b/trunk/src/core/srs_core_version6.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 6 #define VERSION_MINOR 0 -#define VERSION_REVISION 2 +#define VERSION_REVISION 3 #endif diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 95849d02dc..df35dd54a9 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -614,13 +614,22 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size) return srs_error_wrap(err, "add frame"); } -#ifdef SRS_H265 SrsVideoCodecConfig* c = vcodec(); - bool parse_nalus = !c || c->id == SrsVideoCodecIdAVC || c->id == SrsVideoCodecIdForbidden; - if (!parse_nalus) return err; + if (size <= 0) return err; + + // For HEVC(H.265), try to parse the IDR from NALUs. + if (c && c->id == SrsVideoCodecIdHEVC) { +#ifdef SRS_H265 + SrsHevcNaluType nalu_type = (SrsHevcNaluType)(uint8_t)((bytes[0] & 0x3f) >> 1); + has_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23); + return err; +#else + return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); #endif - - // for video, parse the nalu type, set the IDR flag. + } + + // By default, use AVC(H.264) to parse NALU. + // For video, parse the nalu type, set the IDR flag. SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(bytes[0] & 0x1f); if (nal_unit_type == SrsAvcNaluTypeIDR) { @@ -932,7 +941,19 @@ srs_error_t SrsFormat::hevc_demux_hvcc(SrsBuffer* stream) dec_conf_rec_p->constant_frame_rate = (data_byte >> 6) & 0x03; dec_conf_rec_p->num_temporal_layers = (data_byte >> 3) & 0x07; dec_conf_rec_p->temporal_id_nested = (data_byte >> 2) & 0x01; + + // Parse the NALU size. dec_conf_rec_p->length_size_minus_one = data_byte & 0x03; + vcodec->NAL_unit_length = dec_conf_rec_p->length_size_minus_one; + + // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16 + // 5.2.4.1 AVC decoder configuration record + // 5.2.4.1.2 Semantics + // The value of this field shall be one of 0, 1, or 3 corresponding to a + // length encoded with 1, 2, or 4 bytes, respectively. + if (vcodec->NAL_unit_length == 2) { + return srs_error_new(ERROR_HLS_DECODE_ERROR, "sps lengthSizeMinusOne should never be 2"); + } uint8_t numOfArrays = stream->read_1bytes(); srs_info("avg_frame_rate:%d, constant_frame_rate:%d, num_temporal_layers:%d, temporal_id_nested:%d, length_size_minus_one:%d, numOfArrays:%d", @@ -972,70 +993,6 @@ srs_error_t SrsFormat::hevc_demux_hvcc(SrsBuffer* stream) return srs_success; } - -srs_error_t SrsFormat::hevc_demux_ibmf_format(SrsBuffer* stream) -{ - srs_error_t err = srs_success; - - int PictureLength = stream->size() - stream->pos(); - int nal_len_size = vcodec->hevc_dec_conf_record_.length_size_minus_one; - - // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16 - // 5.2.4.1 AVC decoder configuration record - // 5.2.4.1.2 Semantics - // The value of this field shall be one of 0, 1, or 3 corresponding to a - // length encoded with 1, 2, or 4 bytes, respectively. - srs_assert(nal_len_size != 2); - - // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 - for (int i = 0; i < PictureLength;) { - if (i + nal_len_size >= PictureLength) { - break; - } - // unsigned int((NAL_unit_length+1)*8) NALUnitLength; - if (!stream->require(nal_len_size + 1)) { - return srs_error_new(ERROR_HLS_DECODE_ERROR, "nal_len_size:%d, PictureLength:%d, i:%d", - nal_len_size, PictureLength, i); - } - int32_t NALUnitLength = 0; - - if (nal_len_size == 3) { - NALUnitLength = stream->read_4bytes(); - } else if (nal_len_size == 1) { - NALUnitLength = stream->read_2bytes(); - } else { - NALUnitLength = stream->read_1bytes(); - } - - // maybe stream is invalid format. - // see: https://github.com/ossrs/srs/issues/183 - if (NALUnitLength < 0) { - return srs_error_new(ERROR_HLS_DECODE_ERROR, "pic length:%d, NAL_unit_length:%d, NALUnitLength:%d", - PictureLength, nal_len_size, NALUnitLength); - } - - // NALUnit - if (!stream->require(NALUnitLength)) { - return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU data"); - } - - uint8_t* header_p = (uint8_t*)(stream->data() + stream->pos()); - uint8_t nalu_type = (*header_p & 0x3f) >> 1; - bool irap = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23); - if (irap) { - video->has_idr = true; - } - - if ((err = video->add_sample(stream->data() + stream->pos(), NALUnitLength)) != srs_success) { - return srs_error_wrap(err, "avc add video frame"); - } - stream->skip(NALUnitLength); - - i += vcodec->NAL_unit_length + 1 + NALUnitLength; - } - - return err; -} #endif srs_error_t SrsFormat::avc_demux_sps_pps(SrsBuffer* stream) @@ -1411,12 +1368,14 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) return err; } -#ifdef SRS_H265 if (vcodec->id == SrsVideoCodecIdHEVC) { +#ifdef SRS_H265 // TODO: FIXME: Might need to guess format? - return hevc_demux_ibmf_format(stream); - } + return do_avc_demux_ibmf_format(stream); +#else + return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); #endif + } // Parse the SPS/PPS in ANNEXB or IBMF format. if (vcodec->payload_format == SrsAvcPayloadFormatIbmf) { @@ -1542,7 +1501,8 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream) for (int i = 0; i < PictureLength;) { // unsigned int((NAL_unit_length+1)*8) NALUnitLength; if (!stream->require(vcodec->NAL_unit_length + 1)) { - return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU size"); + return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d", + PictureLength, i, vcodec->NAL_unit_length, stream->left()); } int32_t NALUnitLength = 0; if (vcodec->NAL_unit_length == 3) { @@ -1553,22 +1513,23 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream) NALUnitLength = stream->read_1bytes(); } - // maybe stream is invalid format. - // see: https://github.com/ossrs/srs/issues/183 + // The stream format mighe be incorrect, see: https://github.com/ossrs/srs/issues/183 if (NALUnitLength < 0) { - return srs_error_new(ERROR_HLS_DECODE_ERROR, "maybe stream is AnnexB format"); + return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d, NALUnitLength:%d", + PictureLength, i, vcodec->NAL_unit_length, stream->left(), NALUnitLength); } // NALUnit if (!stream->require(NALUnitLength)) { - return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU data"); + return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d, NALUnitLength:%d", + PictureLength, i, vcodec->NAL_unit_length, stream->left(), NALUnitLength); } // 7.3.1 NAL unit syntax, ISO_IEC_14496-10-AVC-2003.pdf, page 44. if ((err = video->add_sample(stream->data() + stream->pos(), NALUnitLength)) != srs_success) { return srs_error_wrap(err, "avc add video frame"); } + stream->skip(NALUnitLength); - i += vcodec->NAL_unit_length + 1 + NALUnitLength; } diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 41ad19bf1a..aa7b32c51d 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -873,7 +873,6 @@ class SrsFormat #ifdef SRS_H265 private: virtual srs_error_t hevc_demux_hvcc(SrsBuffer* stream); - virtual srs_error_t hevc_demux_ibmf_format(SrsBuffer* stream); #endif private: // Parse the H.264 SPS/PPS. diff --git a/trunk/src/utest/srs_utest_kernel.cpp b/trunk/src/utest/srs_utest_kernel.cpp index b3ec8e6951..a469a61069 100644 --- a/trunk/src/utest/srs_utest_kernel.cpp +++ b/trunk/src/utest/srs_utest_kernel.cpp @@ -3525,6 +3525,10 @@ VOID TEST(KernelCodecTest, AVFrame) if (true) { SrsVideoFrame f; + SrsVideoCodecConfig cc; + HELPER_EXPECT_SUCCESS(f.initialize(&cc)); + EXPECT_TRUE(f.vcodec() != NULL); + for (int i = 0; i < SrsMaxNbSamples; i++) { HELPER_EXPECT_SUCCESS(f.add_sample((char*)"\x05", 1)); } @@ -3534,6 +3538,21 @@ VOID TEST(KernelCodecTest, AVFrame) } } +VOID TEST(KernelCodecTest, AVFrameNoConfig) +{ + srs_error_t err; + + if (true) { + SrsAudioFrame f; + HELPER_EXPECT_SUCCESS(f.add_sample((char*)1, 10)); + } + + if (true) { + SrsVideoFrame f; + HELPER_EXPECT_SUCCESS(f.add_sample((char*)"\x05", 1)); + } +} + VOID TEST(KernelCodecTest, IsSequenceHeaderSpecial) { if (true) {