From d91f5e34a828381f95647657912cb88431f4b918 Mon Sep 17 00:00:00 2001 From: runner365 Date: Fri, 3 Jul 2020 11:06:05 +0800 Subject: [PATCH] H265: For #1747, Support HEVC/H.265 in SRT/RTMP/HLS. --- trunk/auto/auto_headers.sh | 6 + trunk/auto/options.sh | 5 + trunk/configure | 3 + trunk/src/app/srs_app_hls.cpp | 8 +- trunk/src/app/srs_app_source.cpp | 7 +- trunk/src/kernel/srs_kernel_codec.cpp | 384 +++++++++++++-- trunk/src/kernel/srs_kernel_codec.hpp | 140 +++++- trunk/src/kernel/srs_kernel_error.hpp | 5 + trunk/src/kernel/srs_kernel_ts.cpp | 214 ++++++--- trunk/src/kernel/srs_kernel_ts.hpp | 8 + trunk/src/protocol/srs_raw_hevc.cpp | 645 ++++++++++++++++++++++++++ trunk/src/protocol/srs_raw_hevc.hpp | 75 +++ trunk/src/srt/srt_to_rtmp.cpp | 196 +++++++- trunk/src/srt/srt_to_rtmp.hpp | 22 +- trunk/src/srt/ts_demux.cpp | 2 +- 15 files changed, 1602 insertions(+), 118 deletions(-) create mode 100644 trunk/src/protocol/srs_raw_hevc.cpp create mode 100644 trunk/src/protocol/srs_raw_hevc.hpp diff --git a/trunk/auto/auto_headers.sh b/trunk/auto/auto_headers.sh index 744c8ab0ec..6933355ff8 100755 --- a/trunk/auto/auto_headers.sh +++ b/trunk/auto/auto_headers.sh @@ -80,6 +80,12 @@ else srs_undefine_macro "SRS_FFMPEG_FIT" $SRS_AUTO_HEADERS_H fi +if [ $SRS_H265 = YES ]; then + srs_define_macro "SRS_H265" $SRS_AUTO_HEADERS_H +else + srs_undefine_macro "SRS_H265" $SRS_AUTO_HEADERS_H +fi + if [ $SRS_SIMULATOR = YES ]; then srs_define_macro "SRS_SIMULATOR" $SRS_AUTO_HEADERS_H else diff --git a/trunk/auto/options.sh b/trunk/auto/options.sh index 4ef8a7e1ae..f75d015f86 100755 --- a/trunk/auto/options.sh +++ b/trunk/auto/options.sh @@ -6,6 +6,7 @@ help=no SRS_HDS=NO SRS_SRT=NO SRS_RTC=YES +SRS_H265=YES SRS_GB28181=NO SRS_ICONV=NO SRS_CXX11=YES @@ -128,6 +129,7 @@ Features: --cxx11=on|off Whether enable the C++11. Default: $(value2switch $SRS_CXX11) --cxx14=on|off Whether enable the C++14. Default: $(value2switch $SRS_CXX14) --ffmpeg-fit=on|off Whether enable the FFmpeg fit(source code). Default: $(value2switch $SRS_FFMPEG_FIT) + --h265=on|off Whether build the H.265 support. Default: $(value2switch $SRS_H265) --prefix= The absolute installation path. Default: $SRS_PREFIX --gcov=on|off Whether enable the GCOV compiler options. Default: $(value2switch $SRS_GCOV) @@ -279,6 +281,8 @@ function parse_user_option() { --simulator) SRS_SIMULATOR=$(switch2value $value) ;; --ffmpeg-fit) SRS_FFMPEG_FIT=$(switch2value $value) ;; + --h265) if [[ $value == off ]]; then SRS_H265=NO; else SRS_H265=YES; fi ;; + --with-gb28181) SRS_GB28181=YES ;; --without-gb28181) SRS_GB28181=NO ;; --with-iconv) SRS_ICONV=YES ;; @@ -499,6 +503,7 @@ function regenerate_options() { SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --cherrypy=$(value2switch $SRS_CHERRYPY)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --srt=$(value2switch $SRS_SRT)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --rtc=$(value2switch $SRS_RTC)" + SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --h265=$(value2switch $SRS_H265)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --simulator=$(value2switch $SRS_SIMULATOR)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --gb28181=$(value2switch $SRS_GB28181)" SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --iconv=$(value2switch $SRS_ICONV)" diff --git a/trunk/configure b/trunk/configure index 3356ad3409..fb5a31fc49 100755 --- a/trunk/configure +++ b/trunk/configure @@ -240,6 +240,9 @@ fi if [[ $SRS_FFMPEG_FIT == YES ]]; then ModuleLibIncs+=("${LibFfmpegRoot[*]}") fi +if [[ $SRS_H265 == YES ]]; then + MODULE_FILES+=("srs_raw_hevc") +fi PROTOCOL_INCS="src/protocol"; MODULE_DIR=${PROTOCOL_INCS} . auto/modules.sh PROTOCOL_OBJS="${MODULE_OBJS[@]}" # diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 836dbd7e16..cbf2e2a3aa 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -367,6 +367,8 @@ srs_error_t SrsHlsMuxer::segment_open() std::string default_vcodec_str = _srs_config->get_hls_vcodec(req->vhost); if (default_vcodec_str == "h264") { default_vcodec = SrsVideoCodecIdAVC; + } else if (default_vcodec_str == "h265") { + default_vcodec = SrsVideoCodecIdHEVC; } else if (default_vcodec_str == "vn") { default_vcodec = SrsVideoCodecIdDisabled; } else { @@ -1308,7 +1310,11 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma } srs_assert(format->vcodec); - if (format->vcodec->id != SrsVideoCodecIdAVC) { + bool ignore_frame = (format->vcodec->id != SrsVideoCodecIdAVC); +#ifdef SRS_H265 + ignore_frame = ignore_frame && (format->vcodec->id != SrsVideoCodecIdHEVC); +#endif + if (ignore_frame) { return err; } diff --git a/trunk/src/app/srs_app_source.cpp b/trunk/src/app/srs_app_source.cpp index d3d26e7b64..df4baebb3e 100755 --- a/trunk/src/app/srs_app_source.cpp +++ b/trunk/src/app/srs_app_source.cpp @@ -611,8 +611,11 @@ srs_error_t SrsGopCache::cache(SrsSharedPtrMessage* shared_msg) // got video, update the video count if acceptable if (msg->is_video()) { - // drop video when not h.264 - if (!SrsFlvVideo::h264(msg->payload, msg->size)) { + bool ignore_frame = !SrsFlvVideo::h264(msg->payload, msg->size); +#ifdef SRS_H265 + ignore_frame = ignore_frame && !SrsFlvVideo::hevc(msg->payload, msg->size); +#endif + if (ignore_frame) { return err; } diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 1bb324c124..a7b35c0802 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -107,8 +107,11 @@ bool SrsFlvVideo::keyframe(char* data, int size) bool SrsFlvVideo::sh(char* data, int size) { - // sequence header only for h264 - if (!h264(data, size)) { + bool ignore_frame = !h264(data, size); +#ifdef SRS_H265 + ignore_frame = ignore_frame && !hevc(data, size); +#endif + if (ignore_frame) { return false; } @@ -139,6 +142,21 @@ bool SrsFlvVideo::h264(char* data, int size) return codec_id == SrsVideoCodecIdAVC; } +#ifdef SRS_H265 +bool SrsFlvVideo::hevc(char* data, int size) +{ + // 1bytes required. + if (size < 1) { + return false; + } + + char codec_id = data[0]; + codec_id = codec_id & 0x0F; + + return codec_id == SrsVideoCodecIdHEVC; +} +#endif + bool SrsFlvVideo::acceptable(char* data, int size) { // 1bytes required. @@ -153,8 +171,12 @@ bool SrsFlvVideo::acceptable(char* data, int size) if (frame_type < 1 || frame_type > 5) { return false; } - - if (codec_id < 2 || codec_id > 7) { + + bool ignore_frame = codec_id < 2; +#ifdef SRS_H265 + ignore_frame = ignore_frame || codec_id > SrsVideoCodecIdHEVC; +#endif + if (ignore_frame) { return false; } @@ -541,21 +563,25 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size) if ((err = SrsFrame::add_sample(bytes, size)) != srs_success) { return srs_error_wrap(err, "add frame"); } - - // for video, parse the nalu type, set the IDR flag. - SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(bytes[0] & 0x1f); - - if (nal_unit_type == SrsAvcNaluTypeIDR) { - has_idr = true; - } else if (nal_unit_type == SrsAvcNaluTypeSPS || nal_unit_type == SrsAvcNaluTypePPS) { - has_sps_pps = true; - } else if (nal_unit_type == SrsAvcNaluTypeAccessUnitDelimiter) { - has_aud = true; - } - - if (first_nalu_type == SrsAvcNaluTypeReserved) { - first_nalu_type = nal_unit_type; + +#ifdef SRS_H265 + if (vcodec()->id == SrsVideoCodecIdAVC) { + // for video, parse the nalu type, set the IDR flag. + SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(bytes[0] & 0x1f); + + if (nal_unit_type == SrsAvcNaluTypeIDR) { + has_idr = true; + } else if (nal_unit_type == SrsAvcNaluTypeSPS || nal_unit_type == SrsAvcNaluTypePPS) { + has_sps_pps = true; + } else if (nal_unit_type == SrsAvcNaluTypeAccessUnitDelimiter) { + has_aud = true; + } + + if (first_nalu_type == SrsAvcNaluTypeReserved) { + first_nalu_type = nal_unit_type; + } } +#endif return err; } @@ -651,9 +677,13 @@ srs_error_t SrsFormat::on_video(int64_t timestamp, char* data, int size) // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 int8_t frame_type = buffer->read_1bytes(); SrsVideoCodecId codec_id = (SrsVideoCodecId)(frame_type & 0x0f); - + // TODO: Support other codecs. - if (codec_id != SrsVideoCodecIdAVC) { + bool ignore_frame = (codec_id != SrsVideoCodecIdAVC); +#ifdef SRS_H265 + ignore_frame = ignore_frame && (codec_id != SrsVideoCodecIdHEVC); +#endif + if (ignore_frame) { return err; } @@ -723,8 +753,12 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp) return err; } - // only support h.264/avc - if (codec_id != SrsVideoCodecIdAVC) { + // support h.264/avc + bool ignore_frame = (codec_id != SrsVideoCodecIdAVC); +#ifdef SRS_H265 + ignore_frame = ignore_frame && (codec_id != SrsVideoCodecIdHEVC); +#endif + if (ignore_frame) { return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc only support video h.264/avc, actual=%d", codec_id); } vcodec->id = codec_id; @@ -743,23 +777,173 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp) // Update the RAW AVC data. raw = stream->data() + stream->pos(); nb_raw = stream->size() - stream->pos(); - - if (avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { - // TODO: FIXME: Maybe we should ignore any error for parsing sps/pps. - if ((err = avc_demux_sps_pps(stream)) != srs_success) { - return srs_error_wrap(err, "demux SPS/PPS"); + + if (codec_id == SrsVideoCodecIdAVC) { + if (avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { + // TODO: FIXME: Maybe we should ignore any error for parsing sps/pps. + if ((err = avc_demux_sps_pps(stream)) != srs_success) { + return srs_error_wrap(err, "demux SPS/PPS"); + } + } else if (avc_packet_type == SrsVideoAvcFrameTraitNALU){ + if ((err = video_nalu_demux(stream)) != srs_success) { + return srs_error_wrap(err, "demux NALU"); + } + } else { + // ignored. } - } else if (avc_packet_type == SrsVideoAvcFrameTraitNALU){ - if ((err = video_nalu_demux(stream)) != srs_success) { - return srs_error_wrap(err, "demux NALU"); + } + +#ifdef SRS_H265 + // analyze hevc here + if (codec_id == SrsVideoCodecIdHEVC) { + if (avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { + // TODO: demux vps/sps/pps for hevc + if ((err = hevc_demux_hvcc(stream)) != srs_success) { + return srs_error_wrap(err, "demux hevc SPS/PPS"); + } + } else if (avc_packet_type == SrsVideoAvcFrameTraitNALU) { + // TODO: demux nalu for hevc + if ((err = video_nalu_demux(stream)) != srs_success) { + return srs_error_wrap(err, "demux hevc NALU"); + } + } else { + // ignore } - } else { - // ignored. } +#endif return err; } +#ifdef SRS_H265 +// Parse the hevc vps/sps/pps +srs_error_t SrsFormat::hevc_demux_hvcc(SrsBuffer* stream) { + const int HEVC_MIN_SIZE = 23;//from configurationVersion to numOfArrays + uint8_t data_byte; + uint64_t data_64bit; + uint8_t numOfArrays; + + int avc_extra_size = stream->size() - stream->pos(); + if (avc_extra_size > 0) { + char *copy_stream_from = stream->data() + stream->pos(); + vcodec->avc_extra_data = std::vector(copy_stream_from, copy_stream_from + avc_extra_size); + } + + if (!stream->require(HEVC_MIN_SIZE)) { + return srs_error_new(ERROR_HLS_DECODE_ERROR, "hevc decode sequence header"); + } + HEVCDecoderConfigurationRecord* dec_conf_rec_p = &(vcodec->_hevcDecConfRecord); + + dec_conf_rec_p->configurationVersion = stream->read_1bytes(); + if (dec_conf_rec_p->configurationVersion != 1) { + return srs_error_new(ERROR_HLS_DECODE_ERROR, "hevc decode sequence header"); + } + + //general_profile_space(2bits), general_tier_flag(1bit), general_profile_idc(5bits) + data_byte = stream->read_1bytes(); + dec_conf_rec_p->general_profile_space = (data_byte >> 6) & 0x03; + dec_conf_rec_p->general_tier_flag = (data_byte >> 5) & 0x01; + dec_conf_rec_p->general_profile_idc = data_byte & 0x1F; + + srs_info("hevc version:%d, general_profile_space:%d, general_tier_flag:%d, general_profile_idc:%d", + dec_conf_rec_p->configurationVersion, + dec_conf_rec_p->general_profile_space, + dec_conf_rec_p->general_tier_flag, + dec_conf_rec_p->general_profile_idc); + //general_profile_compatibility_flags: 32bits + dec_conf_rec_p->general_profile_compatibility_flags = (uint32_t)stream->read_4bytes(); + + //general_constraint_indicator_flags: 48bits + data_64bit = (uint64_t)stream->read_4bytes(); + data_64bit = (data_64bit << 16) | (stream->read_2bytes()); + dec_conf_rec_p->general_constraint_indicator_flags = data_64bit; + + //general_level_idc: 8bits + dec_conf_rec_p->general_level_idc = stream->read_1bytes(); + + //min_spatial_segmentation_idc: xxxx 14bits + dec_conf_rec_p->min_spatial_segmentation_idc = stream->read_2bytes() & 0x0fff; + + //parallelismType: xxxx xx 2bits + dec_conf_rec_p->parallelismType = stream->read_1bytes() & 0x03; + + //chromaFormat: xxxx xx 2bits + dec_conf_rec_p->chromaFormat = stream->read_1bytes() & 0x03; + + //bitDepthLumaMinus8: xxxx x 3bits + dec_conf_rec_p->bitDepthLumaMinus8 = stream->read_1bytes() & 0x07; + + //bitDepthChromaMinus8: xxxx x 3bits + dec_conf_rec_p->bitDepthChromaMinus8 = stream->read_1bytes() & 0x07; + + srs_info("general_constraint_indicator_flags:0x%x, general_level_idc:%d, \ +min_spatial_segmentation_idc:%d, parallelismType:%d, chromaFormat:%d, bitDepthLumaMinus8:%d, \ +bitDepthChromaMinus8:%d", + dec_conf_rec_p->general_constraint_indicator_flags, + dec_conf_rec_p->general_level_idc, + dec_conf_rec_p->min_spatial_segmentation_idc, + dec_conf_rec_p->parallelismType, + dec_conf_rec_p->chromaFormat, + dec_conf_rec_p->bitDepthLumaMinus8, + dec_conf_rec_p->bitDepthChromaMinus8); + + //avgFrameRate: 16bits + dec_conf_rec_p->avgFrameRate = stream->read_2bytes(); + //8bits: constantFrameRate(2bits), numTemporalLayers(3bits), + // temporalIdNested(1bit), lengthSizeMinusOne(2bits) + data_byte = stream->read_1bytes(); + dec_conf_rec_p->constantFrameRate = (data_byte >> 6) & 0x03; + dec_conf_rec_p->numTemporalLayers = (data_byte >> 3) & 0x07; + dec_conf_rec_p->temporalIdNested = (data_byte >> 2) & 0x01; + dec_conf_rec_p->lengthSizeMinusOne = data_byte & 0x03; + + numOfArrays = stream->read_1bytes(); + + srs_info("avgFrameRate:%d, constantFrameRate:%d, numTemporalLayers:%d, \ +temporalIdNested:%d, lengthSizeMinusOne:%d, numOfArrays:%d", + dec_conf_rec_p->avgFrameRate, + dec_conf_rec_p->constantFrameRate, + dec_conf_rec_p->numTemporalLayers, + + dec_conf_rec_p->temporalIdNested, + dec_conf_rec_p->lengthSizeMinusOne, + numOfArrays); + + //parse vps/pps/sps + for (int index = 0; index < numOfArrays; index++) { + HVCCNALUnit hevc_unit; + + if (!stream->require(5)) { + return srs_error_new(ERROR_HLS_DECODE_ERROR, "hevc decode sequence header"); + } + data_byte = stream->read_1bytes(); + hevc_unit.array_completeness = (data_byte >> 7) & 0x01; + hevc_unit.NAL_unit_type = data_byte & 0x3f; + hevc_unit.numNalus = stream->read_2bytes(); + + for (int i = 0; i < hevc_unit.numNalus; i++) { + HEVCNalData data_item; + data_item.nalUnitLength = stream->read_2bytes(); + + if (!stream->require(data_item.nalUnitLength)) { + return srs_error_new(ERROR_HLS_DECODE_ERROR, "hevc decode sequence header"); + } + //copy vps/pps/sps data + data_item.nalUnitData.resize(data_item.nalUnitLength); + + stream->read_bytes((char*)(&data_item.nalUnitData[0]), + data_item.nalUnitLength); + srs_info("hevc nalu type:%d, array_completeness:%d, numNalus:%d, i:%d, nalUnitLength:%d", + hevc_unit.NAL_unit_type, hevc_unit.array_completeness, + hevc_unit.numNalus, i, data_item.nalUnitLength); + hevc_unit.nalData_vec.push_back(data_item); + } + dec_conf_rec_p->nalu_vec.push_back(hevc_unit); + } + return srs_success; +} +#endif + // For media server, we don't care the codec, so we just try to parse sps-pps, and we could ignore any error if fail. // LCOV_EXCL_START @@ -1082,7 +1266,17 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) srs_warn("avc ignore type=%d for no sequence header", SrsVideoAvcFrameTraitNALU); return err; } - + + if (vcodec->id == SrsVideoCodecIdAVC) { + demux_ibmf_format_func = &SrsFormat::avc_demux_ibmf_format; +#ifdef SRS_H265 + } else if (vcodec->id == SrsVideoCodecIdHEVC) { + demux_ibmf_format_func = &SrsFormat::hevc_demux_ibmf_format; +#endif + } else { + return srs_error_wrap(err, "avc demux ibmf"); + } + // guess for the first time. if (vcodec->payload_format == SrsAvcPayloadFormatGuess) { // One or more NALUs (Full frames are required) @@ -1095,7 +1289,7 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) srs_freep(err); // try "ISO Base Media File Format" from ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 - if ((err = avc_demux_ibmf_format(stream)) != srs_success) { + if ((err = (this->*demux_ibmf_format_func)(stream)) != srs_success) { return srs_error_wrap(err, "avc demux ibmf"); } else { vcodec->payload_format = SrsAvcPayloadFormatIbmf; @@ -1105,7 +1299,7 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) } } else if (vcodec->payload_format == SrsAvcPayloadFormatIbmf) { // try "ISO Base Media File Format" from ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 - if ((err = avc_demux_ibmf_format(stream)) != srs_success) { + if ((err = (this->*demux_ibmf_format_func)(stream)) != srs_success) { return srs_error_wrap(err, "avc demux ibmf"); } } else { @@ -1119,7 +1313,8 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) srs_freep(err); // try "ISO Base Media File Format" from ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 - if ((err = avc_demux_ibmf_format(stream)) != srs_success) { + if ((err = (this->*demux_ibmf_format_func) + (stream)) != srs_success) { return srs_error_wrap(err, "avc demux ibmf"); } else { vcodec->payload_format = SrsAvcPayloadFormatIbmf; @@ -1130,6 +1325,123 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) return err; } +#ifdef SRS_H265 +srs_error_t SrsFormat::hevc_vps_data(char*& data_p, int& len) { + srs_error_t err = srs_success; + + for (size_t index = 0; index < vcodec->_hevcDecConfRecord.nalu_vec.size(); index++) { + HVCCNALUnit nalu_unit = vcodec->_hevcDecConfRecord.nalu_vec[index]; + + if (nalu_unit.NAL_unit_type == NAL_UNIT_VPS) { + if (nalu_unit.nalData_vec.size() > 0) { + data_p = (char*)(&nalu_unit.nalData_vec[0].nalUnitData[0]); + len = nalu_unit.nalData_vec[0].nalUnitLength; + } + } + } + return err; +} + +srs_error_t SrsFormat::hevc_sps_data(char*& data_p, int& len) { + srs_error_t err = srs_success; + + for (size_t index = 0; index < vcodec->_hevcDecConfRecord.nalu_vec.size(); index++) { + HVCCNALUnit nalu_unit = vcodec->_hevcDecConfRecord.nalu_vec[index]; + + if (nalu_unit.NAL_unit_type == NAL_UNIT_SPS) { + if (nalu_unit.nalData_vec.size() > 0) { + data_p = (char*)(&nalu_unit.nalData_vec[0].nalUnitData[0]); + len = nalu_unit.nalData_vec[0].nalUnitLength; + } + } + } + return err; +} + +srs_error_t SrsFormat::hevc_pps_data(char*& data_p, int& len) { + srs_error_t err = srs_success; + + for (size_t index = 0; index < vcodec->_hevcDecConfRecord.nalu_vec.size(); index++) { + HVCCNALUnit nalu_unit = vcodec->_hevcDecConfRecord.nalu_vec[index]; + + if (nalu_unit.NAL_unit_type == NAL_UNIT_PPS) { + if (nalu_unit.nalData_vec.size() > 0) { + data_p = (char*)(&nalu_unit.nalData_vec[0].nalUnitData[0]); + len = nalu_unit.nalData_vec[0].nalUnitLength; + } + } + } + return err; +} + +srs_error_t SrsFormat::hevc_demux_ibmf_format(SrsBuffer* stream) { + srs_error_t err = srs_success; + int PictureLength = stream->size() - stream->pos(); + int nal_len_size = 0; + + nal_len_size = vcodec->_hevcDecConfRecord.lengthSizeMinusOne; + + // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16 + // 5.2.4.1 AVC decoder configuration record + // 5.2.4.1.2 Semantics + // The value of this field shall be one of 0, 1, or 3 corresponding to a + // length encoded with 1, 2, or 4 bytes, respectively. + srs_assert(nal_len_size != 2); + + // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 + for (int i = 0; i < PictureLength;) { + if (i + nal_len_size >= PictureLength) { + break; + } + // unsigned int((NAL_unit_length+1)*8) NALUnitLength; + if (!stream->require(nal_len_size + 1)) { + srs_error("nal_len_size:%d, PictureLength:%d, i:%d, 0x%02x.", + nal_len_size, PictureLength, i); + return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU size"); + } + int32_t NALUnitLength = 0; + + if (nal_len_size == 3) { + NALUnitLength = stream->read_4bytes(); + } else if (nal_len_size == 1) { + NALUnitLength = stream->read_2bytes(); + } else { + NALUnitLength = stream->read_1bytes(); + } + + // maybe stream is invalid format. + // see: https://github.com/ossrs/srs/issues/183 + if (NALUnitLength < 0) { + srs_error("pic length:%d, NAL_unit_length:%d, NALUnitLength:%d", + PictureLength, nal_len_size, NALUnitLength); + return srs_error_new(ERROR_HLS_DECODE_ERROR, "maybe stream is AnnexB format"); + } + + // NALUnit + if (!stream->require(NALUnitLength)) { + return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode NALU data"); + } + + uint8_t* header_p = (uint8_t*)(stream->data() + stream->pos()); + uint8_t nalu_type = (*header_p & 0x3f) >> 1; + bool irap = (NAL_UNIT_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= NAL_UNIT_RESERVED_23); + + if (irap) { + video->has_idr = true; + } + + if ((err = video->add_sample(stream->data() + stream->pos(), NALUnitLength)) != srs_success) { + return srs_error_wrap(err, "avc add video frame"); + } + stream->skip(NALUnitLength); + + i += vcodec->NAL_unit_length + 1 + NALUnitLength; + } + + return err; +} +#endif + srs_error_t SrsFormat::avc_demux_annexb_format(SrsBuffer* stream) { srs_error_t err = srs_success; diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 179d280d50..5c952462d9 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -25,6 +25,7 @@ class SrsBuffer; * 5 = On2 VP6 with alpha channel * 6 = Screen video version 2 * 7 = AVC + * 12 = HEVC */ enum SrsVideoCodecId { @@ -245,6 +246,12 @@ class SrsFlvVideo * check codec h264. */ static bool h264(char* data, int size); +#ifdef SRS_H265 + /** + * check codec hevc. + */ + static bool hevc(char* data, int size); +#endif /** * check the video RTMP/flv header info, * @return true if video RTMP/flv header is ok. @@ -381,6 +388,80 @@ enum SrsAvcNaluType }; std::string srs_avc_nalu2str(SrsAvcNaluType nalu_type); +#ifdef SRS_H265 +enum SrsHevcNaluType +{ + NAL_UNIT_CODED_SLICE_TRAIL_N = 0, + NAL_UNIT_CODED_SLICE_TRAIL_R, //1 + NAL_UNIT_CODED_SLICE_TSA_N, //2 + NAL_UNIT_CODED_SLICE_TLA, //3 + NAL_UNIT_CODED_SLICE_STSA_N, //4 + NAL_UNIT_CODED_SLICE_STSA_R, //5 + NAL_UNIT_CODED_SLICE_RADL_N, //6 + NAL_UNIT_CODED_SLICE_DLP, //7 + NAL_UNIT_CODED_SLICE_RASL_N, //8 + NAL_UNIT_CODED_SLICE_TFD, //9 + NAL_UNIT_RESERVED_10, + NAL_UNIT_RESERVED_11, + NAL_UNIT_RESERVED_12, + NAL_UNIT_RESERVED_13, + NAL_UNIT_RESERVED_14, + NAL_UNIT_RESERVED_15, + NAL_UNIT_CODED_SLICE_BLA, //16 + NAL_UNIT_CODED_SLICE_BLANT, //17 + NAL_UNIT_CODED_SLICE_BLA_N_LP, //18 + NAL_UNIT_CODED_SLICE_IDR, //19 + NAL_UNIT_CODED_SLICE_IDR_N_LP, //20 + NAL_UNIT_CODED_SLICE_CRA, //21 + NAL_UNIT_RESERVED_22, + NAL_UNIT_RESERVED_23, + NAL_UNIT_RESERVED_24, + NAL_UNIT_RESERVED_25, + NAL_UNIT_RESERVED_26, + NAL_UNIT_RESERVED_27, + NAL_UNIT_RESERVED_28, + NAL_UNIT_RESERVED_29, + NAL_UNIT_RESERVED_30, + NAL_UNIT_RESERVED_31, + NAL_UNIT_VPS, // 32 + NAL_UNIT_SPS, // 33 + NAL_UNIT_PPS, // 34 + NAL_UNIT_ACCESS_UNIT_DELIMITER, // 35 + NAL_UNIT_EOS, // 36 + NAL_UNIT_EOB, // 37 + NAL_UNIT_FILLER_DATA, // 38 + NAL_UNIT_SEI , // 39 Prefix SEI + NAL_UNIT_SEI_SUFFIX, // 40 Suffix SEI + NAL_UNIT_RESERVED_41, + NAL_UNIT_RESERVED_42, + NAL_UNIT_RESERVED_43, + NAL_UNIT_RESERVED_44, + NAL_UNIT_RESERVED_45, + NAL_UNIT_RESERVED_46, + NAL_UNIT_RESERVED_47, + NAL_UNIT_UNSPECIFIED_48, + NAL_UNIT_UNSPECIFIED_49, + NAL_UNIT_UNSPECIFIED_50, + NAL_UNIT_UNSPECIFIED_51, + NAL_UNIT_UNSPECIFIED_52, + NAL_UNIT_UNSPECIFIED_53, + NAL_UNIT_UNSPECIFIED_54, + NAL_UNIT_UNSPECIFIED_55, + NAL_UNIT_UNSPECIFIED_56, + NAL_UNIT_UNSPECIFIED_57, + NAL_UNIT_UNSPECIFIED_58, + NAL_UNIT_UNSPECIFIED_59, + NAL_UNIT_UNSPECIFIED_60, + NAL_UNIT_UNSPECIFIED_61, + NAL_UNIT_UNSPECIFIED_62, + NAL_UNIT_UNSPECIFIED_63, + NAL_UNIT_INVALID, +}; + +//for nalu data first byte +#define HEVC_NALU_TYPE(code) (SrsHevcNaluType)((code & 0x7E)>>1) +#endif + /** * Table 7-6 – Name association to slice_type * ISO_IEC_14496-10-AVC-2012.pdf, page 105. @@ -591,6 +672,41 @@ class SrsAudioCodecConfig : public SrsCodecConfig virtual bool is_aac_codec_ok(); }; +#ifdef SRS_H265 +struct HEVCNalData { + uint16_t nalUnitLength; + std::vector nalUnitData; +}; + +struct HVCCNALUnit { + uint8_t array_completeness; + uint8_t NAL_unit_type; + uint16_t numNalus; + std::vector nalData_vec; +}; + +struct HEVCDecoderConfigurationRecord { + uint8_t configurationVersion; + uint8_t general_profile_space; + uint8_t general_tier_flag; + uint8_t general_profile_idc; + uint32_t general_profile_compatibility_flags; + uint64_t general_constraint_indicator_flags; + uint8_t general_level_idc; + uint16_t min_spatial_segmentation_idc; + uint8_t parallelismType; + uint8_t chromaFormat; + uint8_t bitDepthLumaMinus8; + uint8_t bitDepthChromaMinus8; + uint16_t avgFrameRate; + uint8_t constantFrameRate; + uint8_t numTemporalLayers; + uint8_t temporalIdNested; + uint8_t lengthSizeMinusOne; + std::vector nalu_vec; +}; +#endif + /** * The video codec info. */ @@ -610,7 +726,7 @@ class SrsVideoCodecConfig : public SrsCodecConfig * @see: ffmpeg, AVCodecContext::extradata */ std::vector avc_extra_data; -public: +public://H264 /** * video specified */ @@ -626,6 +742,10 @@ class SrsVideoCodecConfig : public SrsCodecConfig public: // the avc payload format. SrsAvcPayloadFormat payload_format; +#ifdef SRS_H265 +public: + HEVCDecoderConfigurationRecord _hevcDecConfRecord; +#endif public: SrsVideoCodecConfig(); virtual ~SrsVideoCodecConfig(); @@ -747,10 +867,24 @@ class SrsFormat virtual srs_error_t avc_demux_sps_pps(SrsBuffer* stream); virtual srs_error_t avc_demux_sps(); virtual srs_error_t avc_demux_sps_rbsp(char* rbsp, int nb_rbsp); + +private: + srs_error_t (SrsFormat::*demux_ibmf_format_func)(SrsBuffer* stream); + +#ifdef SRS_H265 +private: + // Parse the hevc vps/sps/pps + virtual srs_error_t hevc_demux_hvcc(SrsBuffer* stream); + virtual srs_error_t hevc_demux_ibmf_format(SrsBuffer* stream); + virtual srs_error_t hevc_vps_data(char*& data_p, int& len); + virtual srs_error_t hevc_pps_data(char*& data_p, int& len); + virtual srs_error_t hevc_sps_data(char*& data_p, int& len); +#endif + private: - // Parse the H.264 NALUs. + // Parse the H.264/hevc NALUs. virtual srs_error_t video_nalu_demux(SrsBuffer* stream); - // Demux the avc NALU in "AnnexB" from ISO_IEC_14496-10-AVC-2003.pdf, page 211. + // Demux the avc/hevc NALU in "AnnexB" from ISO_IEC_14496-10-AVC-2003.pdf, page 211. virtual srs_error_t avc_demux_annexb_format(SrsBuffer* stream); // Demux the avc NALU in "ISO Base Media File Format" from ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 virtual srs_error_t avc_demux_ibmf_format(SrsBuffer* stream); diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 2a4433dabf..62f4aa9d37 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -314,6 +314,11 @@ #define ERROR_HTTPS_READ 4043 #define ERROR_HTTPS_WRITE 4044 #define ERROR_HTTPS_KEY_CRT 4045 +#define ERROR_STREAM_CASTER_HEVC_PPS 4046 +#define ERROR_STREAM_CASTER_HEVC_VPS 4047 +#define ERROR_STREAM_CASTER_HEVC_SPS 4048 +#define ERROR_HEVC_API_NO_PREFIXED 4049 +#define ERROR_HEVC_DROP_BEFORE_SPS_PPS 4050 /////////////////////////////////////////////////////// // RTC protocol error. diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 9d63cc4147..ab752346da 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -43,6 +43,9 @@ string srs_ts_stream2string(SrsTsStream stream) case SrsTsStreamAudioAC3: return "AC3"; case SrsTsStreamAudioDTS: return "AudioDTS"; case SrsTsStreamVideoH264: return "H.264"; +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: return "H.265"; +#endif case SrsTsStreamVideoMpeg4: return "MP4"; case SrsTsStreamAudioMpeg4: return "MP4A"; default: return "Other"; @@ -78,6 +81,9 @@ SrsTsMessage::SrsTsMessage(SrsTsChannel* c, SrsTsPacket* p) start_pts = 0; write_pcr = false; + + //default is h264, which can be update based on the real video stream. + _id = SrsVideoCodecIdAVC; } SrsTsMessage::~SrsTsMessage() @@ -277,11 +283,18 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr SrsTsStream vs, as; int16_t video_pid = 0, audio_pid = 0; + switch (vc) { case SrsVideoCodecIdAVC: vs = SrsTsStreamVideoH264; video_pid = TS_VIDEO_AVC_PID; break; +#ifdef SRS_H265 + case SrsVideoCodecIdHEVC: + vs = SrsTsStreamVideoHEVC; + video_pid = TS_VIDEO_AVC_PID; + break; +#endif case SrsVideoCodecIdDisabled: vs = SrsTsStreamReserved; break; @@ -293,7 +306,6 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr case SrsVideoCodecIdOn2VP6: case SrsVideoCodecIdOn2VP6WithAlphaChannel: case SrsVideoCodecIdScreenVideoVersion2: - case SrsVideoCodecIdHEVC: case SrsVideoCodecIdAV1: vs = SrsTsStreamReserved; break; @@ -357,8 +369,12 @@ void SrsTsContext::set_sync_byte(int8_t sb) srs_error_t SrsTsContext::encode_pat_pmt(ISrsStreamWriter* writer, int16_t vpid, SrsTsStream vs, int16_t apid, SrsTsStream as) { srs_error_t err = srs_success; - - if (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3) { + + bool invalid_codec = (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3); +#ifdef SRS_H265 + invalid_codec = invalid_codec && (vs != SrsTsStreamVideoHEVC); +#endif + if (invalid_codec) { return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no PID, vs=%d, as=%d", vs, as); } @@ -427,8 +443,12 @@ srs_error_t SrsTsContext::encode_pes(ISrsStreamWriter* writer, SrsTsMessage* msg if (msg->payload->length() == 0) { return err; } - - if (sid != SrsTsStreamVideoH264 && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC) { + + bool invalid_codec = (sid != SrsTsStreamVideoH264 && && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC); +#ifdef SRS_H265 + invalid_codec = invalid_codec && (sid != SrsTsStreamVideoHEVC); +#endif + if (invalid_codec) { srs_info("ts: ignore the unknown stream, sid=%d", sid); return err; } @@ -754,7 +774,11 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context, pmt->last_section_number = 0; // must got one valid codec. - srs_assert(vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); + bool is_valid_codec = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); +#ifdef SRS_H265 + is_valid_codec = is_valid_codec || vs == SrsTsStreamVideoHEVC; +#endif + srs_assert(is_valid_codec); // if mp3 or aac specified, use audio to carry pcr. if (as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3) { @@ -765,7 +789,11 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context, } // if h.264 specified, use video to carry pcr. - if (vs == SrsTsStreamVideoH264) { + bool is_valid_codec = (vs == SrsTsStreamVideoH264); +#ifdef SRS_H265 + is_valid_codec = is_valid_codec || (vs == SrsTsStreamVideoHEVC); +#endif + if (is_valid_codec) { pmt->PCR_PID = vpid; pmt->infos.push_back(new SrsTsPayloadPMTESInfo(vs, vpid)); } @@ -2490,6 +2518,9 @@ srs_error_t SrsTsPayloadPMT::psi_decode(SrsBuffer* stream) // update the apply pid table switch (info->stream_type) { case SrsTsStreamVideoH264: +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: +#endif case SrsTsStreamVideoMpeg4: packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type); break; @@ -2573,6 +2604,9 @@ srs_error_t SrsTsPayloadPMT::psi_encode(SrsBuffer* stream) // update the apply pid table switch (info->stream_type) { case SrsTsStreamVideoH264: +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: +#endif case SrsTsStreamVideoMpeg4: packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type); break; @@ -2626,9 +2660,9 @@ srs_error_t SrsTsContextWriter::write_video(SrsTsMessage* video) { srs_error_t err = srs_success; - srs_info("hls: write video pts=%" PRId64 ", dts=%" PRId64 ", size=%d", - video->pts, video->dts, video->PES_packet_length); - + vcodec = video->_id;//update vcodec based on video frame + srs_info("hls: write video pts=%" PRId64 ", dts=%" PRId64 ", size=%d, codec:%d", + video->pts, video->dts, video->PES_packet_length, vcodec); if ((err = context->encode(writer, video, vcodec, acodec)) != srs_success) { return srs_error_wrap(err, "ts: write video"); } @@ -2642,6 +2676,10 @@ SrsVideoCodecId SrsTsContextWriter::video_codec() return vcodec; } +void SrsTsContextWriter::update_vcodec(SrsVideoCodecId id) { + vcodec = id; +} + SrsEncFileWriter::SrsEncFileWriter() { memset(iv,0,16); @@ -2789,7 +2827,8 @@ srs_error_t SrsTsMessageCache::cache_video(SrsVideoFrame* frame, int64_t dts) video->dts = dts; video->pts = video->dts + frame->cts * 90; video->sid = SrsTsPESStreamIdVideoCommon; - + video->_id = frame->vcodec()->id; + // write video to cache. if ((err = do_cache_avc(frame)) != srs_success) { return srs_error_wrap(err, "ts: cache avc"); @@ -2940,49 +2979,50 @@ void srs_avc_insert_aud(SrsSimpleStream* payload, bool& aud_inserted) srs_error_t SrsTsMessageCache::do_cache_avc(SrsVideoFrame* frame) { srs_error_t err = srs_success; - + SrsVideoCodecConfig* codec = frame->vcodec(); // Whether aud inserted. bool aud_inserted = false; - // Insert a default AUD NALU when no AUD in samples. - if (!frame->has_aud) { - // the aud(access unit delimiter) before each frame. - // 7.3.2.4 Access unit delimiter RBSP syntax - // ISO_IEC_14496-10-AVC-2012.pdf, page 66. - // - // primary_pic_type u(3), the first 3bits, primary_pic_type indicates that the slice_type values - // for all slices of the primary coded picture are members of the set listed in Table 7-5 for - // the given value of primary_pic_type. - // 0, slice_type 2, 7 - // 1, slice_type 0, 2, 5, 7 - // 2, slice_type 0, 1, 2, 5, 6, 7 - // 3, slice_type 4, 9 - // 4, slice_type 3, 4, 8, 9 - // 5, slice_type 2, 4, 7, 9 - // 6, slice_type 0, 2, 3, 4, 5, 7, 8, 9 - // 7, slice_type 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 - // 7.4.2.4 Access unit delimiter RBSP semantics - // ISO_IEC_14496-10-AVC-2012.pdf, page 102. - // - // slice_type specifies the coding type of the slice according to Table 7-6. - // 0, P (P slice) - // 1, B (B slice) - // 2, I (I slice) - // 3, SP (SP slice) - // 4, SI (SI slice) - // 5, P (P slice) - // 6, B (B slice) - // 7, I (I slice) - // 8, SP (SP slice) - // 9, SI (SI slice) - // ISO_IEC_14496-10-AVC-2012.pdf, page 105. - static uint8_t default_aud_nalu[] = { 0x09, 0xf0}; - srs_avc_insert_aud(video->payload, aud_inserted); - video->payload->append((const char*)default_aud_nalu, 2); - } - - SrsVideoCodecConfig* codec = frame->vcodec(); srs_assert(codec); + + if (codec->id == SrsVideoCodecIdAVC) { + // Insert a default AUD NALU when no AUD in samples. + if (!frame->has_aud) { + // the aud(access unit delimiter) before each frame. + // 7.3.2.4 Access unit delimiter RBSP syntax + // ISO_IEC_14496-10-AVC-2012.pdf, page 66. + // + // primary_pic_type u(3), the first 3bits, primary_pic_type indicates that the slice_type values + // for all slices of the primary coded picture are members of the set listed in Table 7-5 for + // the given value of primary_pic_type. + // 0, slice_type 2, 7 + // 1, slice_type 0, 2, 5, 7 + // 2, slice_type 0, 1, 2, 5, 6, 7 + // 3, slice_type 4, 9 + // 4, slice_type 3, 4, 8, 9 + // 5, slice_type 2, 4, 7, 9 + // 6, slice_type 0, 2, 3, 4, 5, 7, 8, 9 + // 7, slice_type 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 + // 7.4.2.4 Access unit delimiter RBSP semantics + // ISO_IEC_14496-10-AVC-2012.pdf, page 102. + // + // slice_type specifies the coding type of the slice according to Table 7-6. + // 0, P (P slice) + // 1, B (B slice) + // 2, I (I slice) + // 3, SP (SP slice) + // 4, SI (SI slice) + // 5, P (P slice) + // 6, B (B slice) + // 7, I (I slice) + // 8, SP (SP slice) + // 9, SI (SI slice) + // ISO_IEC_14496-10-AVC-2012.pdf, page 105. + static uint8_t default_aud_nalu[] = { 0x09, 0xf0}; + srs_avc_insert_aud(video->payload, aud_inserted); + video->payload->append((const char*)default_aud_nalu, 2); + } + } bool is_sps_pps_appended = false; @@ -2995,29 +3035,55 @@ srs_error_t SrsTsMessageCache::do_cache_avc(SrsVideoFrame* frame) return srs_error_new(ERROR_HLS_AVC_SAMPLE_SIZE, "ts: invalid avc sample length=%d", size); } - // 5bits, 7.3.1 NAL unit syntax, - // ISO_IEC_14496-10-AVC-2012.pdf, page 83. - SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(sample->bytes[0] & 0x1f); - - // Insert sps/pps before IDR when there is no sps/pps in samples. - // The sps/pps is parsed from sequence header(generally the first flv packet). - if (nal_unit_type == SrsAvcNaluTypeIDR && !frame->has_sps_pps && !is_sps_pps_appended) { - if (!codec->sequenceParameterSetNALUnit.empty()) { - srs_avc_insert_aud(video->payload, aud_inserted); - video->payload->append(&codec->sequenceParameterSetNALUnit[0], (int)codec->sequenceParameterSetNALUnit.size()); + if (codec->id == SrsVideoCodecIdAVC) { + // 5bits, 7.3.1 NAL unit syntax, + // ISO_IEC_14496-10-AVC-2012.pdf, page 83. + SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(sample->bytes[0] & 0x1f); + + // Insert sps/pps before IDR when there is no sps/pps in samples. + // The sps/pps is parsed from sequence header(generally the first flv packet). + if (nal_unit_type == SrsAvcNaluTypeIDR && !frame->has_sps_pps && !is_sps_pps_appended) { + if (!codec->sequenceParameterSetNALUnit.empty()) { + srs_avc_insert_aud(video->payload, aud_inserted); + video->payload->append(&codec->sequenceParameterSetNALUnit[0], (int)codec->sequenceParameterSetNALUnit.size()); + } + if (!codec->pictureParameterSetNALUnit.empty()) { + srs_avc_insert_aud(video->payload, aud_inserted); + video->payload->append(&codec->pictureParameterSetNALUnit[0], (int)codec->pictureParameterSetNALUnit.size()); + } + is_sps_pps_appended = true; } - if (!codec->pictureParameterSetNALUnit.empty()) { - srs_avc_insert_aud(video->payload, aud_inserted); - video->payload->append(&codec->pictureParameterSetNALUnit[0], (int)codec->pictureParameterSetNALUnit.size()); + + // Insert the NALU to video in annexb. + srs_avc_insert_aud(video->payload, aud_inserted); + video->payload->append(sample->bytes, sample->size); + } + +#ifdef SRS_H265 + if (codec->id == SrsVideoCodecIdHEVC) { + const char start_code[] = {0, 0, 0, 1}; + SrsHevcNaluType hevc_nalu_type = (SrsHevcNaluType)HEVC_NALU_TYPE(sample->bytes[0]); + + if ((hevc_nalu_type >= NAL_UNIT_CODED_SLICE_BLA) && (hevc_nalu_type <= NAL_UNIT_RESERVED_23) + && !frame->has_sps_pps && !is_sps_pps_appended) { + for (size_t i = 0; i < codec->_hevcDecConfRecord.nalu_vec.size(); i++) { + HVCCNALUnit item = codec->_hevcDecConfRecord.nalu_vec[i]; + + if ((item.numNalus > 0) && (item.nalData_vec.size() > 0)) { + video->payload->append(start_code, sizeof(start_code)); + video->payload->append((char*)&item.nalData_vec[0].nalUnitData[0], + (int)item.nalData_vec[0].nalUnitData.size()); + is_sps_pps_appended = true; + } + } } - is_sps_pps_appended = true; + // Insert the NALU to video in annexb. + video->payload->append(start_code, sizeof(start_code)); + video->payload->append(sample->bytes, sample->size); } - - // Insert the NALU to video in annexb. - srs_avc_insert_aud(video->payload, aud_inserted); - video->payload->append(sample->bytes, sample->size); +#endif } - + return err; } @@ -3109,14 +3175,20 @@ srs_error_t SrsTsTransmuxer::write_video(int64_t timestamp, char* data, int size return err; } + tscw->update_vcodec(format->vcodec->id);//update vcodec id by video frame. + // ignore info frame, // @see https://github.com/ossrs/srs/issues/288#issuecomment-69863909 srs_assert(format->video && format->vcodec); if (format->video->frame_type == SrsVideoAvcFrameTypeVideoInfoFrame) { return err; } - - if (format->vcodec->id != SrsVideoCodecIdAVC) { + + bool invalid_codec = (format->vcodec->id != SrsVideoCodecIdAVC); +#ifdef SRS_H265 + invalid_codec = invalid_codec && (format->vcodec->id != SrsVideoCodecIdHEVC); +#endif + if (invalid_codec) { return err; } diff --git a/trunk/src/kernel/srs_kernel_ts.hpp b/trunk/src/kernel/srs_kernel_ts.hpp index e8f09ef2ec..a536b11e1d 100644 --- a/trunk/src/kernel/srs_kernel_ts.hpp +++ b/trunk/src/kernel/srs_kernel_ts.hpp @@ -130,6 +130,10 @@ enum SrsTsStream // ITU-T Rec. H.222.0 | ISO/IEC 13818-1 Reserved // 0x15-0x7F SrsTsStreamVideoH264 = 0x1b, +#ifdef SRS_H265 + SrsTsStreamVideoHEVC = 0x24, +#endif + // User Private // 0x80-0xFF SrsTsStreamAudioAC3 = 0x81, @@ -242,6 +246,9 @@ class SrsTsMessage uint8_t continuity_counter; // The payload bytes. SrsSimpleStream* payload; +public: + SrsVideoCodecId _id; + public: SrsTsMessage(SrsTsChannel* c = NULL, SrsTsPacket* p = NULL); virtual ~SrsTsMessage(); @@ -1259,6 +1266,7 @@ class SrsTsContextWriter public: // get the video codec of ts muxer. virtual SrsVideoCodecId video_codec(); + virtual void update_vcodec(SrsVideoCodecId id); }; // Used for HLS Encryption diff --git a/trunk/src/protocol/srs_raw_hevc.cpp b/trunk/src/protocol/srs_raw_hevc.cpp new file mode 100644 index 0000000000..0539b965da --- /dev/null +++ b/trunk/src/protocol/srs_raw_hevc.cpp @@ -0,0 +1,645 @@ +#include "srs_raw_hevc.hpp" +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +#define MAX(x, y) ((x) > (y) ? (x) : (y)) + +#define BIT(ptr, off) (((ptr)[(off) / 8] >> (7 - ((off) % 8))) & 0x01) + +struct mpeg4_hevc_t +{ + uint8_t configurationVersion; // 1-only + uint8_t general_profile_space; // 2bit,[0,3] + uint8_t general_tier_flag; // 1bit,[0,1] + uint8_t general_profile_idc; // 5bit,[0,31] + uint32_t general_profile_compatibility_flags; + uint64_t general_constraint_indicator_flags; + uint8_t general_level_idc; + uint16_t min_spatial_segmentation_idc; + uint8_t parallelismType; // 2bit,[0,3] + uint8_t chromaFormat; // 2bit,[0,3] + uint8_t bitDepthLumaMinus8; // 3bit,[0,7] + uint8_t bitDepthChromaMinus8; // 3bit,[0,7] + uint16_t avgFrameRate; + uint8_t constantFrameRate; // 2bit,[0,3] + uint8_t numTemporalLayers; // 3bit,[0,7] + uint8_t temporalIdNested; // 1bit,[0,1] + uint8_t lengthSizeMinusOne; // 2bit,[0,3] + + uint8_t numOfArrays; + struct + { + uint8_t array_completeness; + uint8_t type; // nalu type + uint16_t bytes; + uint8_t* data; + } nalu[64]; + + uint8_t array_completeness; + uint8_t data[4 * 1024]; + int off; +}; + +static uint8_t mpeg4_h264_read_ue(const uint8_t* data, int bytes, int* offset) +{ + int bit, i; + int leadingZeroBits = -1; + + for (bit = 0; !bit && *offset / 8 < bytes; ++leadingZeroBits) + { + bit = (data[*offset / 8] >> (7 - (*offset % 8))) & 0x01; + ++*offset; + } + + bit = 0; + assert(leadingZeroBits < 32); + for (i = 0; i < leadingZeroBits && *offset / 8 < bytes; i++) + { + bit = (bit << 1) | ((data[*offset / 8] >> (7 - (*offset % 8))) & 0x01); + ++*offset; + } + + return (uint8_t)((1 << leadingZeroBits) - 1 + bit); +} + +static int hevc_rbsp_decode(const uint8_t* nalu, int bytes, uint8_t* sodb) +{ + int i, j; + for (j = i = 0; i < bytes; i++) + { + if (i + 2 < bytes && 0 == nalu[i] && 0 == nalu[i + 1] && 0x03 == nalu[i + 2]) + { + sodb[j++] = nalu[i]; + sodb[j++] = nalu[i + 1]; + i += 2; + } + else + { + sodb[j++] = nalu[i]; + } + } + return j; +} + +static int hevc_profile_tier_level(const uint8_t* nalu, int bytes, uint8_t maxNumSubLayersMinus1, struct mpeg4_hevc_t* hevc) +{ + int n; + uint8_t i; + uint8_t sub_layer_profile_present_flag[8]; + uint8_t sub_layer_level_present_flag[8]; + + if (bytes < 12) + return -1; + + hevc->general_profile_space = (nalu[0] >> 6) & 0x03; + hevc->general_tier_flag = (nalu[0] >> 5) & 0x01; + hevc->general_profile_idc = nalu[0] & 0x1f; + + hevc->general_profile_compatibility_flags = 0; + hevc->general_profile_compatibility_flags |= nalu[1] << 24; + hevc->general_profile_compatibility_flags |= nalu[2] << 16; + hevc->general_profile_compatibility_flags |= nalu[3] << 8; + hevc->general_profile_compatibility_flags |= nalu[4]; + + hevc->general_constraint_indicator_flags = 0; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[5]) << 40; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[6]) << 32; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[7]) << 24; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[8]) << 16; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[9]) << 8; + hevc->general_constraint_indicator_flags |= nalu[10]; + + hevc->general_level_idc = nalu[11]; + if (maxNumSubLayersMinus1 < 1) + return 12; + + if (bytes < 14) + return -1; // error + + for (i = 0; i < maxNumSubLayersMinus1; i++) + { + sub_layer_profile_present_flag[i] = BIT(nalu, 12 * 8 + i * 2); + sub_layer_level_present_flag[i] = BIT(nalu, 12 * 8 + i * 2 + 1); + } + + n = 12 + 2; + for (i = 0; i < maxNumSubLayersMinus1; i++) + { + if(sub_layer_profile_present_flag[i]) + n += 11; + if (sub_layer_level_present_flag[i]) + n += 1; + } + + return bytes < n ? n : -1; +} + +static uint8_t hevc_vps_id(const uint8_t* rbsp, int bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr) +{ + int sodb; + uint8_t vps; + uint8_t vps_max_sub_layers_minus1; + uint8_t vps_temporal_id_nesting_flag; + + sodb = hevc_rbsp_decode(rbsp, bytes, ptr); + if (sodb < 16 + 2) + return 0xFF; + + vps = ptr[2] >> 4; // 2-nalu type + vps_max_sub_layers_minus1 = (ptr[3] >> 1) & 0x07; + vps_temporal_id_nesting_flag = ptr[3] & 0x01; + hevc->numTemporalLayers = MAX(hevc->numTemporalLayers, vps_max_sub_layers_minus1 + 1); + hevc->temporalIdNested = (hevc->temporalIdNested || vps_temporal_id_nesting_flag) ? 1 : 0; + hevc_profile_tier_level(ptr + 6, sodb - 6, vps_max_sub_layers_minus1, hevc); + + return vps; +} + +static uint8_t hevc_sps_id(const uint8_t* rbsp, int bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr, uint8_t* vps) +{ + int n; + int sodb; + uint8_t sps; + uint8_t sps_max_sub_layers_minus1; + uint8_t sps_temporal_id_nesting_flag; + uint8_t conformance_window_flag; + + sodb = hevc_rbsp_decode(rbsp, bytes, ptr); + if (sodb < 12+3) + return 0xFF; + + *vps = ptr[2] >> 4; // 2-nalu type + sps_max_sub_layers_minus1 = (ptr[2] >> 1) & 0x07; + sps_temporal_id_nesting_flag = ptr[2] & 0x01; + (void)sps_temporal_id_nesting_flag; + n = hevc_profile_tier_level(ptr + 3, sodb - 3, sps_max_sub_layers_minus1, hevc); + if (n <= 0) + return 0xFF; + + n = (n + 3) * 8; + sps = mpeg4_h264_read_ue(ptr, sodb, &n); + hevc->chromaFormat = mpeg4_h264_read_ue(ptr, sodb, &n); + if (3 == hevc->chromaFormat) + n++; + mpeg4_h264_read_ue(ptr, sodb, &n); // pic_width_in_luma_samples + mpeg4_h264_read_ue(ptr, sodb, &n); // pic_height_in_luma_samples + conformance_window_flag = BIT(ptr, n); n++; // conformance_window_flag + if (conformance_window_flag) + { + mpeg4_h264_read_ue(ptr, sodb, &n); // conf_win_left_offset + mpeg4_h264_read_ue(ptr, sodb, &n); // conf_win_right_offset + mpeg4_h264_read_ue(ptr, sodb, &n); // conf_win_top_offset + mpeg4_h264_read_ue(ptr, sodb, &n); // conf_win_bottom_offset + } + hevc->bitDepthLumaMinus8 = mpeg4_h264_read_ue(ptr, sodb, &n); + hevc->bitDepthChromaMinus8 = mpeg4_h264_read_ue(ptr, sodb, &n); + + // TODO: vui_parameters + //mp4->hevc->min_spatial_segmentation_idc; // min_spatial_segmentation_idc + return sps; +} + +static uint8_t hevc_pps_id(const uint8_t* rbsp, int bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr, uint8_t* sps) +{ + // TODO: + //hevc->parallelismType; // entropy_coding_sync_enabled_flag + (void)hevc; + + int sodb; + int offset = 2 * 8; // 2-nalu type + sodb = hevc_rbsp_decode(rbsp, bytes, ptr); + if (sodb < 3) + return 0xFF; + *sps = mpeg4_h264_read_ue(ptr, sodb, &offset); + return mpeg4_h264_read_ue(ptr, sodb, &offset); +} + +static void mpeg4_hevc_remove(struct mpeg4_hevc_t* hevc, uint8_t* ptr, int bytes, const uint8_t* end) +{ + uint8_t i; + assert(ptr >= hevc->data && ptr + bytes <= end && end <= hevc->data + sizeof(hevc->data)); + memmove(ptr, ptr + bytes, end - ptr - bytes); + + for (i = 0; i < hevc->numOfArrays; i++) + { + if (hevc->nalu[i].data > ptr) + hevc->nalu[i].data -= bytes; + } +} + +static int mpeg4_hevc_update2(struct mpeg4_hevc_t* hevc, int i, const uint8_t* nalu, int bytes) +{ + if (bytes == hevc->nalu[i].bytes && 0 == memcmp(nalu, hevc->nalu[i].data, bytes)) + return 0; // do nothing + + if (bytes > hevc->nalu[i].bytes && hevc->off + (bytes - hevc->nalu[i].bytes) > (int)sizeof(hevc->data)) + { + assert(0); + return -1; // too big + } + + mpeg4_hevc_remove(hevc, hevc->nalu[i].data, hevc->nalu[i].bytes, hevc->data + hevc->off); + hevc->off -= hevc->nalu[i].bytes; + + hevc->nalu[i].data = hevc->data + hevc->off; + hevc->nalu[i].bytes = (uint16_t)bytes; + memcpy(hevc->nalu[i].data, nalu, bytes); + hevc->off += bytes; + return 1; +} + +static int mpeg4_hevc_add(struct mpeg4_hevc_t* hevc, uint8_t type, const uint8_t* nalu, int bytes) +{ + // copy new + assert(hevc->numOfArrays < (int)sizeof(hevc->nalu) / (int)sizeof(hevc->nalu[0])); + if (hevc->numOfArrays >= (int)sizeof(hevc->nalu) / (int)sizeof(hevc->nalu[0]) + || hevc->off + bytes > (int)sizeof(hevc->data)) + { + assert(0); + return -1; + } + + hevc->nalu[hevc->numOfArrays].type = type; + hevc->nalu[hevc->numOfArrays].bytes = (uint16_t)bytes; + hevc->nalu[hevc->numOfArrays].array_completeness = 1; + hevc->nalu[hevc->numOfArrays].data = hevc->data + hevc->off; + memcpy(hevc->nalu[hevc->numOfArrays].data, nalu, bytes); + hevc->off += bytes; + ++hevc->numOfArrays; + return 1; +} + +static int h265_vps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, int bytes) +{ + int i; + uint8_t vpsid; + + if (bytes < 3) + { + assert(0); + return -1; // invalid length + } + + vpsid = hevc_vps_id(nalu, bytes, hevc, hevc->data + hevc->off); + for (i = 0; i < hevc->numOfArrays; i++) + { + if (NAL_UNIT_VPS == hevc->nalu[i].type && vpsid == hevc_vps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, hevc, hevc->data + hevc->off)) + return mpeg4_hevc_update2(hevc, i, nalu, bytes); + } + + return mpeg4_hevc_add(hevc, NAL_UNIT_VPS, nalu, bytes); +} + +static int h265_sps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, int bytes) +{ + int i; + uint8_t spsid; + uint8_t vpsid, vpsid2; + + if (bytes < 13 + 2) + { + assert(0); + return -1; // invalid length + } + + spsid = hevc_sps_id(nalu, bytes, hevc, hevc->data + hevc->off, &vpsid); + for (i = 0; i < hevc->numOfArrays; i++) + { + if (NAL_UNIT_SPS == hevc->nalu[i].type && spsid == hevc_sps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, hevc, hevc->data + hevc->off, &vpsid2) && vpsid == vpsid2) + return mpeg4_hevc_update2(hevc, i, nalu, bytes); + } + + return mpeg4_hevc_add(hevc, NAL_UNIT_SPS, nalu, bytes); +} + +static int h265_pps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, int bytes) +{ + int i; + uint8_t ppsid; + uint8_t spsid, spsid2; + + if (bytes < 1 + 2) + { + assert(0); + return -1; // invalid length + } + + ppsid = hevc_pps_id(nalu, bytes, hevc, hevc->data + hevc->off, &spsid); + for (i = 0; i < hevc->numOfArrays; i++) + { + if (NAL_UNIT_PPS == hevc->nalu[i].type && ppsid == hevc_pps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, hevc, hevc->data + hevc->off, &spsid2) && spsid == spsid2) + return mpeg4_hevc_update2(hevc, i, nalu, bytes); + } + + return mpeg4_hevc_add(hevc, NAL_UNIT_PPS, nalu, bytes); +} + +SrsRawHEVCStream::SrsRawHEVCStream() { + +} + +SrsRawHEVCStream::~SrsRawHEVCStream() { + +} + +srs_error_t SrsRawHEVCStream::annexb_demux(SrsBuffer* stream, char** pframe, int* pnb_frame) +{ + srs_error_t err = srs_success; + + *pframe = NULL; + *pnb_frame = 0; + + while (!stream->empty()) { + // each frame must prefixed by annexb format. + // about annexb, @see ISO_IEC_14496-10-AVC-2003.pdf, page 211. + int pnb_start_code = 0; + if (!srs_avc_startswith_annexb(stream, &pnb_start_code)) { + return srs_error_new(ERROR_HEVC_API_NO_PREFIXED, "hevc annexb start code"); + } + int start = stream->pos() + pnb_start_code; + + // find the last frame prefixed by annexb format. + stream->skip(pnb_start_code); + while (!stream->empty()) { + if (srs_avc_startswith_annexb(stream, NULL)) { + break; + } + stream->skip(1); + } + + // demux the frame. + *pnb_frame = stream->pos() - start; + *pframe = stream->data() + start; + break; + } + + return err; +} + +// whether the frame is sps or pps or vps. +bool SrsRawHEVCStream::is_sps(char* frame, int nb_frame) +{ + srs_assert(nb_frame > 0); + uint8_t nal_unit_type = HEVC_NALU_TYPE(frame[0]); + + return nal_unit_type == NAL_UNIT_SPS; +} + +bool SrsRawHEVCStream::is_pps(char* frame, int nb_frame) { + srs_assert(nb_frame > 0); + uint8_t nal_unit_type = HEVC_NALU_TYPE(frame[0]); + + return nal_unit_type == NAL_UNIT_PPS; +} + +bool SrsRawHEVCStream::is_vps(char* frame, int nb_frame) { + srs_assert(nb_frame > 0); + uint8_t nal_unit_type = HEVC_NALU_TYPE(frame[0]); + + return nal_unit_type == NAL_UNIT_VPS; +} + +srs_error_t SrsRawHEVCStream::sps_demux(char* frame, int nb_frame, std::string& sps) +{ + srs_error_t err = srs_success; + + // atleast 1bytes for SPS to decode the type, profile, constrain and level. + if (nb_frame < 4) { + return err; + } + + sps = string(frame, nb_frame); + + return err; +} + +srs_error_t SrsRawHEVCStream::pps_demux(char* frame, int nb_frame, std::string& pps) +{ + srs_error_t err = srs_success; + + if (nb_frame <= 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_PPS, "no pps"); + } + + pps = string(frame, nb_frame); + + return err; +} + +srs_error_t SrsRawHEVCStream::vps_demux(char* frame, int nb_frame, std::string& vps) +{ + srs_error_t err = srs_success; + + if (nb_frame <= 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_VPS, "no vps"); + } + + vps = string(frame, nb_frame); + + return err; +} + +srs_error_t SrsRawHEVCStream::mux_sequence_header(std::string sps, std::string pps, std::string vps, + uint32_t dts, uint32_t pts, std::string& hvcC) +{ + srs_error_t err = srs_success; + uint8_t temp8bits = 0; + struct mpeg4_hevc_t hevc_info; + + memset(&hevc_info, 0, sizeof(hevc_info)); + + if (h265_vps_copy(&hevc_info, (uint8_t*)vps.data(), (int)vps.length()) < 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_VPS, "decode vps error"); + } + + if (h265_sps_copy(&hevc_info, (uint8_t*)sps.data(), (int)sps.length()) < 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_SPS, "decode sps error"); + } + + if (h265_pps_copy(&hevc_info, (uint8_t*)pps.data(), (int)pps.length()) < 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_PPS, "decode pps error"); + } + + // hevc header information: + // 23bytes header: + // configurationVersion, general_profile_space, general_tier_flag, general_profile_idc + // general_profile_compatibility_flags, general_constraint_indicator_flags, + // general_level_idc, min_spatial_segmentation_idc, parallelismType, + // chromaFormat, bitDepthLumaMinus8, bitDepthChromaMinus8, + // avgFrameRate, constantFrameRate, numTemporalLayers, temporalIdNested, + // lengthSizeMinusOne, numOfArrays + // 5bytes size of vps/sps/pps: + // array_completeness, nal_unit_type, numNalus, nalUnitLength, + // Nbytes of vps/sps/pps. + // sequenceParameterSetNALUnit + + + //use simple mode: nalu size + nalu data + int nb_packet = 23 + 5 + (int)sps.length() + 5 + (int)pps.length() + 5 + (int)vps.length(); + char* packet = new char[nb_packet]; + SrsAutoFreeA(char, packet); + + // use stream to generate the hevc packet. + SrsBuffer stream(packet, nb_packet); + + hevc_info.configurationVersion = 1; + stream.write_1bytes(hevc_info.configurationVersion); + + temp8bits = 0; + temp8bits |= ((hevc_info.general_profile_space << 6) & 0xc0); + temp8bits |= ((hevc_info.general_tier_flag << 5) & 0x20); + temp8bits |= hevc_info.general_profile_idc & 0x1f; + stream.write_1bytes(temp8bits); + + stream.write_4bytes(hevc_info.general_profile_compatibility_flags); + + stream.write_2bytes((hevc_info.general_constraint_indicator_flags >> 32) & 0xffff); + stream.write_4bytes(hevc_info.general_constraint_indicator_flags & 0xffffffff); + + stream.write_1bytes(hevc_info.general_level_idc); + + stream.write_2bytes(0xf000 | (hevc_info.min_spatial_segmentation_idc & 0x0fff)); + + stream.write_1bytes(0xfc|(hevc_info.parallelismType & 0x03)); + stream.write_1bytes(0xfc|(hevc_info.chromaFormat & 0x03)); + + stream.write_1bytes(0xf8|(hevc_info.bitDepthLumaMinus8 & 0x07)); + + stream.write_1bytes(0xf8|(hevc_info.bitDepthChromaMinus8 & 0x07)); + + stream.write_2bytes(hevc_info.avgFrameRate); + + hevc_info.lengthSizeMinusOne = 3; + temp8bits = 0; + temp8bits |= (hevc_info.constantFrameRate << 6) | 0xc0; + temp8bits |= (hevc_info.numTemporalLayers << 3) | 0x38; + temp8bits |= (hevc_info.temporalIdNested << 2) | 0x04; + temp8bits |= hevc_info.lengthSizeMinusOne & 0x03; + + stream.write_1bytes(temp8bits); + + uint8_t numOfArrays = 3;//vps,sps,pps + stream.write_1bytes(numOfArrays); + + uint8_t array_completeness = 0;//1bit + //uint8_t reserved = 0;//1bit + uint8_t nal_unit_type = 0;//6bits; + + //vps + nal_unit_type = ((array_completeness << 7) & 0x80) | (NAL_UNIT_VPS & 0x3f); + stream.write_1bytes(nal_unit_type); + + uint16_t namNalus = 1; + stream.write_2bytes(namNalus); + + uint16_t nalUnitLength = vps.length(); + stream.write_2bytes(nalUnitLength); + + stream.write_string(vps); + + //sps + nal_unit_type = ((array_completeness << 7) & 0x80) | (NAL_UNIT_SPS & 0x3f); + stream.write_1bytes(nal_unit_type); + + namNalus = 1; + stream.write_2bytes(namNalus); + + + nalUnitLength = sps.length(); + stream.write_2bytes(nalUnitLength); + + stream.write_string(sps); + + //pps + nal_unit_type = ((array_completeness << 7) & 0x80) | (NAL_UNIT_PPS & 0x3f); + stream.write_1bytes(nal_unit_type); + + namNalus = 1; + stream.write_2bytes(namNalus); + + nalUnitLength = pps.length(); + stream.write_2bytes(nalUnitLength); + stream.write_string(pps); + + hvcC = string(packet, nb_packet); + + return err; +} + +srs_error_t SrsRawHEVCStream::mux_ipb_frame(char* frame, int nb_frame, std::string& ibp) +{ + srs_error_t err = srs_success; + + // 4bytes size of nalu: + // NALUnitLength + // Nbytes of nalu. + // NALUnit + int nb_packet = 4 + nb_frame; + char* packet = new char[nb_packet]; + SrsAutoFreeA(char, packet); + + // use stream to generate the h264 packet. + SrsBuffer stream(packet, nb_packet); + + // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16 + // lengthSizeMinusOne, or NAL_unit_length, always use 4bytes size + uint32_t NAL_unit_length = nb_frame; + + // mux the avc NALU in "ISO Base Media File Format" + // from ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 + // NALUnitLength + stream.write_4bytes(NAL_unit_length); + // NALUnit + stream.write_bytes(frame, nb_frame); + + ibp = string(packet, nb_packet); + + return err; +} + +srs_error_t SrsRawHEVCStream::mux_avc2flv(std::string video, int8_t frame_type, int8_t avc_packet_type, uint32_t dts, uint32_t pts, char** flv, int* nb_flv) +{ + srs_error_t err = srs_success; + + // for h264 in RTMP video payload, there is 5bytes header: + // 1bytes, FrameType | CodecID + // 1bytes, AVCPacketType + // 3bytes, CompositionTime, the cts. + // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 + int size = (int)video.length() + 5; + char* data = new char[size]; + char* p = data; + + // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 + // Frame Type, Type of video frame. + // CodecID, Codec Identifier. + // set the rtmp header + *p++ = (frame_type << 4) | SrsVideoCodecIdHEVC; + + // AVCPacketType + *p++ = avc_packet_type; + + // CompositionTime + // pts = dts + cts, or + // cts = pts - dts. + // where cts is the header in rtmp video packet payload header. + uint32_t cts = pts - dts; + char* pp = (char*)&cts; + *p++ = pp[2]; + *p++ = pp[1]; + *p++ = pp[0]; + + // hevc raw data. + memcpy(p, video.data(), video.length()); + + *flv = data; + *nb_flv = size; + + return err; +} \ No newline at end of file diff --git a/trunk/src/protocol/srs_raw_hevc.hpp b/trunk/src/protocol/srs_raw_hevc.hpp new file mode 100644 index 0000000000..86e678b098 --- /dev/null +++ b/trunk/src/protocol/srs_raw_hevc.hpp @@ -0,0 +1,75 @@ +/** + * The MIT License (MIT) + * + * Copyright (c) 2013-2020 Winlin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SRS_PROTOCOL_RAW_HEVC_HPP +#define SRS_PROTOCOL_RAW_HEVC_HPP + +#include +#include +#include + +class SrsBuffer; + +// The raw h.264 stream, in annexb. +class SrsRawHEVCStream +{ +public: + SrsRawHEVCStream(); + virtual ~SrsRawHEVCStream(); +public: + // Demux the stream in annexb format. + // @param stream the input stream bytes. + // @param pframe the output hevc frame in stream. user should never free it. + // @param pnb_frame the output hevc frame size. + virtual srs_error_t annexb_demux(SrsBuffer* stream, char** pframe, int* pnb_frame); + // whether the frame is sps or pps or vps. + virtual bool is_sps(char* frame, int nb_frame); + virtual bool is_pps(char* frame, int nb_frame); + virtual bool is_vps(char* frame, int nb_frame); + + // Demux the sps or pps or vps to string. + // @param sps/pps output the sps/pps/vps. + virtual srs_error_t sps_demux(char* frame, int nb_frame, std::string& sps); + virtual srs_error_t pps_demux(char* frame, int nb_frame, std::string& pps); + virtual srs_error_t vps_demux(char* frame, int nb_frame, std::string& vps); +public: + // The hevc raw data to hevc packet, without flv payload header. + // Mux the sps/pps/vps to flv sequence header packet. + // @param sh output the sequence header. + virtual srs_error_t mux_sequence_header(std::string sps, std::string pps, std::string vps, + uint32_t dts, uint32_t pts, std::string& sh); + // The hevc raw data to hevc packet, without flv payload header. + // Mux the ibp to flv ibp packet. + // @param ibp output the packet. + // @param frame_type output the frame type. + virtual srs_error_t mux_ipb_frame(char* frame, int nb_frame, std::string& ibp); + // Mux the hevc video packet to flv video packet. + // @param frame_type, SrsVideoAvcFrameTypeKeyFrame or SrsVideoAvcFrameTypeInterFrame. + // @param avc_packet_type, SrsVideoAvcFrameTraitSequenceHeader or SrsVideoAvcFrameTraitNALU. + // @param video the hevc raw data. + // @param flv output the muxed flv packet. + // @param nb_flv output the muxed flv size. + virtual srs_error_t mux_avc2flv(std::string video, int8_t frame_type, int8_t avc_packet_type, uint32_t dts, uint32_t pts, char** flv, int* nb_flv); +}; + +#endif//end SRS_PROTOCOL_RAW_HEVC_HPP \ No newline at end of file diff --git a/trunk/src/srt/srt_to_rtmp.cpp b/trunk/src/srt/srt_to_rtmp.cpp index 525239166a..625441bdfd 100644 --- a/trunk/src/srt/srt_to_rtmp.cpp +++ b/trunk/src/srt/srt_to_rtmp.cpp @@ -243,6 +243,8 @@ rtmp_client::rtmp_client(std::string key_path):_key_path(key_path) , _connect_flag(false) { const std::string DEF_VHOST = "DEFAULT_VHOST"; _ts_demux_ptr = std::make_shared(); + + _hevc_ptr = std::make_shared(); _avc_ptr = std::make_shared(); _aac_ptr = std::make_shared(); std::vector ret_vec; @@ -285,6 +287,11 @@ rtmp_client::rtmp_client(std::string key_path):_key_path(key_path) _h264_pps_changed = false; _h264_sps_pps_sent = false; + _hevc_vps_changed = false; + _hevc_sps_changed = false; + _hevc_pps_changed = false; + _hevc_sps_pps_sent = false; + _last_live_ts = now_ms(); srs_trace("rtmp client construct url:%s", url_sz); } @@ -348,6 +355,50 @@ void rtmp_client::receive_ts_data(SRT_DATA_MSG_PTR data_ptr) { return; } + +srs_error_t rtmp_client::write_hevc_sps_pps(uint32_t dts, uint32_t pts) { + srs_error_t err = srs_success; + + // TODO: FIMXE: there exists bug, see following comments. + // when sps or pps changed, update the sequence header, + // for the pps maybe not changed while sps changed. + // so, we must check when each video ts message frame parsed. + if (!_hevc_sps_changed || !_hevc_pps_changed || !_hevc_vps_changed) { + return err; + } + + // hevc raw to hevc packet. + std::string sh; + if ((err = _hevc_ptr->mux_sequence_header(_hevc_sps, _hevc_pps, _hevc_vps, dts, pts, sh)) != srs_success) { + return srs_error_wrap(err, "mux sequence header"); + } + + // hevc packet to flv packet. + int8_t frame_type = SrsVideoAvcFrameTypeKeyFrame; + int8_t avc_packet_type = SrsVideoAvcFrameTraitSequenceHeader; + char* flv = NULL; + int nb_flv = 0; + if ((err = _hevc_ptr->mux_avc2flv(sh, frame_type, avc_packet_type, dts, pts, &flv, &nb_flv)) != srs_success) { + return srs_error_wrap(err, "avc to flv"); + } + + if (_srs_config->get_srt_mix_correct()) { + _rtmp_queue.insert_rtmp_data((unsigned char*)flv, nb_flv, (int64_t)dts, SrsFrameTypeVideo); + rtmp_write_work(); + } else { + rtmp_write_packet(SrsFrameTypeVideo, dts, flv, nb_flv); + } + + // reset vps, sps and pps. + _hevc_vps_changed = false; + _hevc_sps_changed = false; + _hevc_pps_changed = false; + + _hevc_sps_pps_sent = true; + + return err; +} + srs_error_t rtmp_client::write_h264_sps_pps(uint32_t dts, uint32_t pts) { srs_error_t err = srs_success; @@ -430,6 +481,43 @@ srs_error_t rtmp_client::write_h264_ipb_frame(char* frame, int frame_size, uint3 return err; } +srs_error_t rtmp_client::write_hevc_ipb_frame(char* frame, int frame_size, uint32_t dts, uint32_t pts) { + srs_error_t err = srs_success; + + // when vps, sps or pps not sent, ignore the packet. + if (!_hevc_sps_pps_sent) { + return srs_error_new(ERROR_HEVC_DROP_BEFORE_SPS_PPS, "drop hevc vps/sps/pps"); + } + + SrsHevcNaluType nal_unit_type = HEVC_NALU_TYPE(frame[0]); + // for IDR frame, the frame is keyframe. + SrsVideoAvcFrameType frame_type = SrsVideoAvcFrameTypeInterFrame; + + if ((nal_unit_type >= NAL_UNIT_CODED_SLICE_BLA) && (nal_unit_type <= NAL_UNIT_RESERVED_23)) { + frame_type = SrsVideoAvcFrameTypeKeyFrame; + } + + std::string ibp; + if ((err = _hevc_ptr->mux_ipb_frame(frame, frame_size, ibp)) != srs_success) { + return srs_error_wrap(err, "mux frame"); + } + + int8_t avc_packet_type = SrsVideoAvcFrameTraitNALU; + char* flv = NULL; + int nb_flv = 0; + if ((err = _hevc_ptr->mux_avc2flv(ibp, frame_type, avc_packet_type, dts, pts, &flv, &nb_flv)) != srs_success) { + return srs_error_wrap(err, "mux avc to flv"); + } + if (_srs_config->get_srt_mix_correct()) { + _rtmp_queue.insert_rtmp_data((unsigned char*)flv, nb_flv, (int64_t)dts, SrsFrameTypeVideo); + rtmp_write_work(); + } else { + rtmp_write_packet(SrsFrameTypeVideo, dts, flv, nb_flv); + } + + return err; +} + srs_error_t rtmp_client::write_audio_raw_frame(char* frame, int frame_size, SrsRawAacStreamCodec* codec, uint32_t dts) { srs_error_t err = srs_success; @@ -490,7 +578,109 @@ srs_error_t rtmp_client::rtmp_write_work() { return err; } -srs_error_t rtmp_client::on_ts_video(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts) { +srs_error_t rtmp_client::on_ts_hevc(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts) { + srs_error_t err = srs_success; + + // ensure rtmp connected. + if ((err = connect()) != srs_success) { + return srs_error_wrap(err, "connect"); + } + dts = dts / 90; + pts = pts / 90; + + if (dts == 0) { + dts = pts; + } + + // send each frame. + while (!avs_ptr->empty()) { + char* frame = NULL; + int frame_size = 0; + if ((err = _hevc_ptr->annexb_demux(avs_ptr.get(), &frame, &frame_size)) != srs_success) { + return srs_error_wrap(err, "demux annexb"); + } + + SrsHevcNaluType nal_unit_type = HEVC_NALU_TYPE(frame[0]); + + if (nal_unit_type == NAL_UNIT_ACCESS_UNIT_DELIMITER) { + continue; + } + // TODO: FIXME: Should cache this config, it's better not to get it for each video frame. + if (_srs_config->get_srt_sei_filter()) { + if (nal_unit_type == NAL_UNIT_SEI) { + continue; + } + } + + // for vps + if (_hevc_ptr->is_vps(frame, frame_size)) { + std::string vps; + if ((err = _hevc_ptr->vps_demux(frame, frame_size, vps)) != srs_success) { + return srs_error_wrap(err, "hevc demux vps"); + } + + if (_hevc_vps == vps) { + continue; + } + _hevc_vps_changed = true; + _hevc_vps = vps; + + //write hevc vps_sps_pps + if ((err = write_hevc_sps_pps(dts, pts)) != srs_success) { + return srs_error_wrap(err, "hevc write vps/sps/pps"); + } + continue; + } + + // for sps + if (_hevc_ptr->is_sps(frame, frame_size)) { + std::string sps; + if ((err = _hevc_ptr->sps_demux(frame, frame_size, sps)) != srs_success) { + return srs_error_wrap(err, "demux sps"); + } + + if (_hevc_sps == sps) { + continue; + } + _hevc_sps_changed = true; + _hevc_sps = sps; + + //write hevc vps_sps_pps + if ((err = write_hevc_sps_pps(dts, pts)) != srs_success) { + return srs_error_wrap(err, "write sps/pps"); + } + continue; + } + + // for pps + if (_hevc_ptr->is_pps(frame, frame_size)) { + std::string pps; + if ((err = _hevc_ptr->pps_demux(frame, frame_size, pps)) != srs_success) { + return srs_error_wrap(err, "demux pps"); + } + + if (_hevc_pps == pps) { + continue; + } + _hevc_pps_changed = true; + _hevc_pps = pps; + + if ((err = write_hevc_sps_pps(dts, pts)) != srs_success) { + return srs_error_wrap(err, "write sps/pps"); + } + continue; + } + + // ibp frame. + // TODO: FIXME: we should group all frames to a rtmp/flv message from one ts message. + write_hevc_ipb_frame(frame, frame_size, dts, pts); + _last_live_ts = now_ms(); + } + + return err; +} + +srs_error_t rtmp_client::on_ts_h264(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts) { srs_error_t err = srs_success; // ensure rtmp connected. @@ -592,7 +782,7 @@ int rtmp_client::get_sample_rate(char sample_index) { return sample_rate; } -srs_error_t rtmp_client::on_ts_audio(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts) { +srs_error_t rtmp_client::on_ts_aac(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts) { srs_error_t err = srs_success; uint64_t base_dts; uint64_t real_dts; @@ -681,6 +871,8 @@ void rtmp_client::on_data_callback(SRT_DATA_MSG_PTR data_ptr, unsigned int media if (media_type == STREAM_TYPE_VIDEO_H264) { err = on_ts_video(avs_ptr, dts, pts); + } else if (media_type == STREAM_TYPE_VIDEO_HEVC) { + on_ts_hevc(avs_ptr, dts, pts); } else if (media_type == STREAM_TYPE_AUDIO_AAC) { err = on_ts_audio(avs_ptr, dts, pts); } else { diff --git a/trunk/src/srt/srt_to_rtmp.hpp b/trunk/src/srt/srt_to_rtmp.hpp index 228a1345cf..3b33d2a8af 100644 --- a/trunk/src/srt/srt_to_rtmp.hpp +++ b/trunk/src/srt/srt_to_rtmp.hpp @@ -19,6 +19,8 @@ #include #include #include +#include + #include #include @@ -31,6 +33,7 @@ typedef std::shared_ptr RTMP_CONN_PTR; typedef std::shared_ptr AVC_PTR; +typedef std::shared_ptr HEVC_PTR; typedef std::shared_ptr AAC_PTR; #define DEFAULT_VHOST "__default_host__" @@ -82,10 +85,14 @@ class rtmp_client : public ts_media_data_callback_I, public std::enable_shared_f virtual void on_data_callback(SRT_DATA_MSG_PTR data_ptr, unsigned int media_type, uint64_t dts, uint64_t pts); private: - srs_error_t on_ts_video(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts); - srs_error_t on_ts_audio(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts); + srs_error_t on_ts_h264(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts); + srs_error_t on_ts_hevc(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts); + srs_error_t on_ts_aac(std::shared_ptr avs_ptr, uint64_t dts, uint64_t pts); + virtual srs_error_t write_h264_sps_pps(uint32_t dts, uint32_t pts); virtual srs_error_t write_h264_ipb_frame(char* frame, int frame_size, uint32_t dts, uint32_t pts); + virtual srs_error_t write_hevc_sps_pps(uint32_t dts, uint32_t pts); + virtual srs_error_t write_hevc_ipb_frame(char* frame, int frame_size, uint32_t dts, uint32_t pts); virtual srs_error_t write_audio_raw_frame(char* frame, int frame_size, SrsRawAacStreamCodec* codec, uint32_t dts); int get_sample_rate(char sound_rate); @@ -110,6 +117,17 @@ class rtmp_client : public ts_media_data_callback_I, public std::enable_shared_f std::string _h264_pps; bool _h264_pps_changed; bool _h264_sps_pps_sent; + +private: + HEVC_PTR _hevc_ptr; + std::string _hevc_vps; + bool _hevc_vps_changed; + std::string _hevc_sps; + bool _hevc_sps_changed; + std::string _hevc_pps; + bool _hevc_pps_changed; + bool _hevc_sps_pps_sent; + private: std::string _aac_specific_config; AAC_PTR _aac_ptr; diff --git a/trunk/src/srt/ts_demux.cpp b/trunk/src/srt/ts_demux.cpp index 575289b5de..a4f3361949 100644 --- a/trunk/src/srt/ts_demux.cpp +++ b/trunk/src/srt/ts_demux.cpp @@ -395,7 +395,7 @@ int ts_demux::pes_parse(unsigned char* p, size_t npos, && stream_id != 248//ITU-T Rec. H.222.1 type E stream 1111 1000 ) { - assert(0x80 == p[pos]); + assert(0x80 == (p[pos] & 0xc0)); //skip 2bits//'10' 2 bslbf int PES_scrambling_control = (p[pos]&30)>>4; //PES_scrambling_control 2 bslbf (void)PES_scrambling_control;