From f618b62f5f443801a08e491966e9d1b926696ce8 Mon Sep 17 00:00:00 2001 From: Michel Promonet Date: Sun, 20 Aug 2023 19:37:26 +0200 Subject: [PATCH] forward frametype --- inc/EncodedVideoFrameBuffer.h | 25 +++++++++++++------------ inc/NullDecoder.h | 16 ++++++++++------ inc/NullEncoder.h | 19 +++++++------------ inc/V4l2Capturer.h | 3 ++- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/inc/EncodedVideoFrameBuffer.h b/inc/EncodedVideoFrameBuffer.h index 077faa09..558f13d4 100644 --- a/inc/EncodedVideoFrameBuffer.h +++ b/inc/EncodedVideoFrameBuffer.h @@ -17,29 +17,30 @@ class EncodedVideoI420Buffer : public webrtc::I420BufferInterface { public: - EncodedVideoI420Buffer(int width, int height, const rtc::scoped_refptr &encoded_data) : width_(width), height_(height), encoded_data_(encoded_data) - { - } - virtual int width() const { return width_; } - virtual int height() const { return height_; } - virtual const uint8_t *DataY() const { return encoded_data_->data(); } + EncodedVideoI420Buffer(int width, int height, const rtc::scoped_refptr &encoded_data, webrtc::VideoFrameType frameType) + : m_width(width), m_height(height), m_encoded_data(encoded_data), m_frameType(frameType) {} + virtual int width() const { return m_width; } + virtual int height() const { return m_height; } + virtual const uint8_t *DataY() const { return m_encoded_data->data(); } virtual const uint8_t *DataU() const { return NULL; } virtual const uint8_t *DataV() const { return NULL; } - virtual int StrideY() const { return encoded_data_->size(); } + virtual int StrideY() const { return m_encoded_data->size(); } virtual int StrideU() const { return 0; } virtual int StrideV() const { return 0; } + webrtc::VideoFrameType getFrameType() const { return m_frameType; } private: - const int width_; - const int height_; - rtc::scoped_refptr encoded_data_; + const int m_width; + const int m_height; + rtc::scoped_refptr m_encoded_data; + webrtc::VideoFrameType m_frameType; }; class EncodedVideoFrameBuffer : public webrtc::VideoFrameBuffer { public: - EncodedVideoFrameBuffer(int width, int height, const rtc::scoped_refptr &encoded_data) - : buffer_(new rtc::RefCountedObject(width, height, encoded_data)) {} + EncodedVideoFrameBuffer(int width, int height, const rtc::scoped_refptr &encoded_data, webrtc::VideoFrameType frameType) + : buffer_(new rtc::RefCountedObject(width, height, encoded_data, frameType)) {} virtual Type type() const { return webrtc::VideoFrameBuffer::Type::kNative; } virtual rtc::scoped_refptr ToI420() { return webrtc::I420Buffer::Create(width(), height()); } virtual int width() const { return buffer_->width(); } diff --git a/inc/NullDecoder.h b/inc/NullDecoder.h index 20dc3e4c..1e7b2e18 100644 --- a/inc/NullDecoder.h +++ b/inc/NullDecoder.h @@ -37,13 +37,17 @@ class NullDecoder : public webrtc::VideoDecoder { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } rtc::scoped_refptr encodedData = input_image.GetEncodedData(); - rtc::scoped_refptr buffer = rtc::make_ref_counted(m_settings.max_render_resolution().Width(), m_settings.max_render_resolution().Height(), encodedData); + rtc::scoped_refptr frameBuffer = rtc::make_ref_counted(m_settings.max_render_resolution().Width(), m_settings.max_render_resolution().Height(), encodedData, input_image._frameType); - webrtc::VideoFrame frame(buffer, webrtc::kVideoRotation_0, render_time_ms * rtc::kNumMicrosecsPerMillisec); - frame.set_timestamp(input_image.Timestamp()); - frame.set_ntp_time_ms(input_image.NtpTimeMs()); - - RTC_LOG(LS_VERBOSE) << "Decode " << frame.id() << " " << input_image._frameType << " " << buffer->width() << "x" << buffer->height() << " " << buffer->GetI420()->StrideY(); + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(frameBuffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_rtp(input_image.Timestamp()) + .set_timestamp_ms(render_time_ms) + .set_ntp_time_ms(input_image.NtpTimeMs()) + .build(); + + RTC_LOG(LS_VERBOSE) << "Decode " << frame.id() << " " << input_image._frameType << " " << frameBuffer->width() << "x" << frameBuffer->height() << " " << frameBuffer->GetI420()->StrideY(); m_decoded_image_callback->Decoded(frame); diff --git a/inc/NullEncoder.h b/inc/NullEncoder.h index fa901728..a26b36ad 100644 --- a/inc/NullEncoder.h +++ b/inc/NullEncoder.h @@ -48,17 +48,10 @@ class NullEncoder : public webrtc::VideoEncoder { } // compute frametype - uint8_t* data = (uint8_t*)buffer->GetI420()->DataY(); - size_t dataSize = buffer->GetI420()->StrideY(); - webrtc::VideoFrameType frameType = webrtc::VideoFrameType::kVideoFrameDelta; - std::vector naluIndexes = webrtc::H264::FindNaluIndices(data, dataSize); - for (webrtc::H264::NaluIndex index : naluIndexes) { - webrtc::H264::NaluType nalu_type = webrtc::H264::ParseNaluType(data[index.payload_start_offset]); - if (nalu_type == webrtc::H264::NaluType::kIdr) { - frameType = webrtc::VideoFrameType::kVideoFrameKey; - break; - } - } + EncodedVideoI420Buffer* encodedBuffer = (EncodedVideoI420Buffer*)buffer->GetI420(); + const uint8_t* data = encodedBuffer->DataY(); + size_t dataSize = encodedBuffer->StrideY(); + webrtc::VideoFrameType frameType = encodedBuffer->getFrameType(); // build webrtc::EncodedImage webrtc::EncodedImage encoded_image; @@ -71,7 +64,9 @@ class NullEncoder : public webrtc::VideoEncoder { // forward to callback webrtc::CodecSpecificInfo codec_specific; - codec_specific.codecType = webrtc::VideoCodecType::kVideoCodecH264; + if (m_format.name == "H264") { + codec_specific.codecType = webrtc::VideoCodecType::kVideoCodecH264; + } webrtc::EncodedImageCallback::Result result = m_encoded_image_callback->OnEncodedImage(encoded_image, &codec_specific); if (result.error == webrtc::EncodedImageCallback::Result::ERROR_SEND_FAILED) { RTC_LOG(LS_ERROR) << "Error in parsing EncodedImage" << encoded_image._frameType; diff --git a/inc/V4l2Capturer.h b/inc/V4l2Capturer.h index edbed227..4e7db572 100644 --- a/inc/V4l2Capturer.h +++ b/inc/V4l2Capturer.h @@ -134,7 +134,8 @@ class V4l2Capturer : public VideoSource } int64_t ts = std::chrono::high_resolution_clock::now().time_since_epoch().count()/1000/1000; - rtc::scoped_refptr frameBuffer = rtc::make_ref_counted(m_capture->getWidth(), m_capture->getHeight(), encodedData); + webrtc::VideoFrameType frameType = idr ? webrtc::VideoFrameType::kVideoFrameKey : webrtc::VideoFrameType::kVideoFrameDelta; + rtc::scoped_refptr frameBuffer = rtc::make_ref_counted(m_capture->getWidth(), m_capture->getHeight(), encodedData, frameType); webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() .set_video_frame_buffer(frameBuffer) .set_rotation(webrtc::kVideoRotation_0)