Skip to content

Commit

Permalink
H265: Support parse multiple NALUs in a frame. v6.0.3 (#3274)
Browse files Browse the repository at this point in the history 
1. Fix parsing multiple NALUs bug.
2. Eliminate duplicated code for parsing NALU.
3. Return error when HEVC not enabled.
  • Loading branch information
@winlinvip
winlinvip authored Nov 23, 2022
1 parent 02d47c5 commit f316e9a
Show file tree
Hide file tree
Showing 6 changed files with 61 additions and 81 deletions.
1 change: 1 addition & 0 deletions trunk/doc/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ The changelog for SRS.

## SRS 6.0 Changelog

* v6.0, 2022-11-23, Merge [#3274](https://github.com/ossrs/srs/pull/3274): H265: Support parse multiple NALUs in a frame. v6.0.3
* v6.0, 2022-11-22, Merge [#3272](https://github.com/ossrs/srs/pull/3272): H265: Support HEVC over RTMP or HTTP-FLV. v6.0.2
* v6.0, 2022-11-22, Merge [#3268](https://github.com/ossrs/srs/pull/3268): H265: Update mpegts.js to play HEVC over HTTP-TS/FLV. v6.0.1
* v6.0, 2022-11-22, Init SRS 6. v6.0.0
Expand Down
2 changes: 1 addition & 1 deletion trunk/ide/srs_clion/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
WORKING_DIRECTORY ${SRS_DIR} RESULT_VARIABLE ret)
ELSE ()
EXECUTE_PROCESS(
COMMAND ./configure --srt=on --gb28181=on --utest=on --jobs=${JOBS}
COMMAND ./configure --srt=on --gb28181=on --h265=on --utest=on --jobs=${JOBS}
WORKING_DIRECTORY ${SRS_DIR} RESULT_VARIABLE ret)
ENDIF ()
if(NOT ret EQUAL 0)
Expand Down
2 changes: 1 addition & 1 deletion trunk/src/core/srs_core_version6.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@

#define VERSION_MAJOR 6
#define VERSION_MINOR 0
#define VERSION_REVISION 2
#define VERSION_REVISION 3

#endif
117 changes: 39 additions & 78 deletions trunk/src/kernel/srs_kernel_codec.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,13 +614,22 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size)
return srs_error_wrap(err, "add frame");
}

#ifdef SRS_H265
SrsVideoCodecConfig* c = vcodec();
bool parse_nalus = !c || c->id == SrsVideoCodecIdAVC || c->id == SrsVideoCodecIdForbidden;
if (!parse_nalus) return err;
if (size <= 0) return err;

// For HEVC(H.265), try to parse the IDR from NALUs.
if (c && c->id == SrsVideoCodecIdHEVC) {
#ifdef SRS_H265
SrsHevcNaluType nalu_type = (SrsHevcNaluType)(uint8_t)((bytes[0] & 0x3f) >> 1);
has_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23);
return err;
#else
return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
#endif

// for video, parse the nalu type, set the IDR flag.
}

// By default, use AVC(H.264) to parse NALU.
// For video, parse the nalu type, set the IDR flag.
SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(bytes[0] & 0x1f);

if (nal_unit_type == SrsAvcNaluTypeIDR) {
Expand Down Expand Up @@ -932,7 +941,19 @@ srs_error_t SrsFormat::hevc_demux_hvcc(SrsBuffer* stream)
dec_conf_rec_p->constant_frame_rate = (data_byte >> 6) & 0x03;
dec_conf_rec_p->num_temporal_layers = (data_byte >> 3) & 0x07;
dec_conf_rec_p->temporal_id_nested = (data_byte >> 2) & 0x01;

// Parse the NALU size.
dec_conf_rec_p->length_size_minus_one = data_byte & 0x03;
vcodec->NAL_unit_length = dec_conf_rec_p->length_size_minus_one;

// 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16
// 5.2.4.1 AVC decoder configuration record
// 5.2.4.1.2 Semantics
// The value of this field shall be one of 0, 1, or 3 corresponding to a
// length encoded with 1, 2, or 4 bytes, respectively.
if (vcodec->NAL_unit_length == 2) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "sps lengthSizeMinusOne should never be 2");
}

uint8_t numOfArrays = stream->read_1bytes();
srs_info("avg_frame_rate:%d, constant_frame_rate:%d, num_temporal_layers:%d, temporal_id_nested:%d, length_size_minus_one:%d, numOfArrays:%d",
Expand Down Expand Up @@ -972,70 +993,6 @@ srs_error_t SrsFormat::hevc_demux_hvcc(SrsBuffer* stream)

return srs_success;
}

srs_error_t SrsFormat::hevc_demux_ibmf_format(SrsBuffer* stream)
{
srs_error_t err = srs_success;

int PictureLength = stream->size() - stream->pos();
int nal_len_size = vcodec->hevc_dec_conf_record_.length_size_minus_one;

// 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16
// 5.2.4.1 AVC decoder configuration record
// 5.2.4.1.2 Semantics
// The value of this field shall be one of 0, 1, or 3 corresponding to a
// length encoded with 1, 2, or 4 bytes, respectively.
srs_assert(nal_len_size != 2);

// 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 20
for (int i = 0; i < PictureLength;) {
if (i + nal_len_size >= PictureLength) {
break;
}
// unsigned int((NAL_unit_length+1)*8) NALUnitLength;
if (!stream->require(nal_len_size + 1)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "nal_len_size:%d, PictureLength:%d, i:%d",
nal_len_size, PictureLength, i);
}
int32_t NALUnitLength = 0;

if (nal_len_size == 3) {
NALUnitLength = stream->read_4bytes();
} else if (nal_len_size == 1) {
NALUnitLength = stream->read_2bytes();
} else {
NALUnitLength = stream->read_1bytes();
}

// maybe stream is invalid format.
// see: https://github.com/ossrs/srs/issues/183
if (NALUnitLength < 0) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "pic length:%d, NAL_unit_length:%d, NALUnitLength:%d",
PictureLength, nal_len_size, NALUnitLength);
}

// NALUnit
if (!stream->require(NALUnitLength)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU data");
}

uint8_t* header_p = (uint8_t*)(stream->data() + stream->pos());
uint8_t nalu_type = (*header_p & 0x3f) >> 1;
bool irap = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23);
if (irap) {
video->has_idr = true;
}

if ((err = video->add_sample(stream->data() + stream->pos(), NALUnitLength)) != srs_success) {
return srs_error_wrap(err, "avc add video frame");
}
stream->skip(NALUnitLength);

i += vcodec->NAL_unit_length + 1 + NALUnitLength;
}

return err;
}
#endif

srs_error_t SrsFormat::avc_demux_sps_pps(SrsBuffer* stream)
Expand Down Expand Up @@ -1411,12 +1368,14 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream)
return err;
}

#ifdef SRS_H265
if (vcodec->id == SrsVideoCodecIdHEVC) {
#ifdef SRS_H265
// TODO: FIXME: Might need to guess format?
return hevc_demux_ibmf_format(stream);
}
return do_avc_demux_ibmf_format(stream);
#else
return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
#endif
}

// Parse the SPS/PPS in ANNEXB or IBMF format.
if (vcodec->payload_format == SrsAvcPayloadFormatIbmf) {
Expand Down Expand Up @@ -1542,7 +1501,8 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream)
for (int i = 0; i < PictureLength;) {
// unsigned int((NAL_unit_length+1)*8) NALUnitLength;
if (!stream->require(vcodec->NAL_unit_length + 1)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU size");
return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d",
PictureLength, i, vcodec->NAL_unit_length, stream->left());
}
int32_t NALUnitLength = 0;
if (vcodec->NAL_unit_length == 3) {
Expand All @@ -1553,22 +1513,23 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream)
NALUnitLength = stream->read_1bytes();
}

// maybe stream is invalid format.
// see: https://github.com/ossrs/srs/issues/183
// The stream format mighe be incorrect, see: https://github.com/ossrs/srs/issues/183
if (NALUnitLength < 0) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "maybe stream is AnnexB format");
return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d, NALUnitLength:%d",
PictureLength, i, vcodec->NAL_unit_length, stream->left(), NALUnitLength);
}

// NALUnit
if (!stream->require(NALUnitLength)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU data");
return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d, NALUnitLength:%d",
PictureLength, i, vcodec->NAL_unit_length, stream->left(), NALUnitLength);
}
// 7.3.1 NAL unit syntax, ISO_IEC_14496-10-AVC-2003.pdf, page 44.
if ((err = video->add_sample(stream->data() + stream->pos(), NALUnitLength)) != srs_success) {
return srs_error_wrap(err, "avc add video frame");
}

stream->skip(NALUnitLength);

i += vcodec->NAL_unit_length + 1 + NALUnitLength;
}

Expand Down
1 change: 0 additions & 1 deletion trunk/src/kernel/srs_kernel_codec.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,6 @@ class SrsFormat
#ifdef SRS_H265
private:
virtual srs_error_t hevc_demux_hvcc(SrsBuffer* stream);
virtual srs_error_t hevc_demux_ibmf_format(SrsBuffer* stream);
#endif
private:
// Parse the H.264 SPS/PPS.
Expand Down
19 changes: 19 additions & 0 deletions trunk/src/utest/srs_utest_kernel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3525,6 +3525,10 @@ VOID TEST(KernelCodecTest, AVFrame)

if (true) {
SrsVideoFrame f;
SrsVideoCodecConfig cc;
HELPER_EXPECT_SUCCESS(f.initialize(&cc));
EXPECT_TRUE(f.vcodec() != NULL);

for (int i = 0; i < SrsMaxNbSamples; i++) {
HELPER_EXPECT_SUCCESS(f.add_sample((char*)"\x05", 1));
}
Expand All @@ -3534,6 +3538,21 @@ VOID TEST(KernelCodecTest, AVFrame)
}
}

VOID TEST(KernelCodecTest, AVFrameNoConfig)
{
srs_error_t err;

if (true) {
SrsAudioFrame f;
HELPER_EXPECT_SUCCESS(f.add_sample((char*)1, 10));
}

if (true) {
SrsVideoFrame f;
HELPER_EXPECT_SUCCESS(f.add_sample((char*)"\x05", 1));
}
}

VOID TEST(KernelCodecTest, IsSequenceHeaderSpecial)
{
if (true) {
Expand Down

0 comments on commit f316e9a

Please sign in to comment.