From f65f2954d51708e597377f044ef715341ed0c4dc Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Tue, 15 Oct 2019 15:09:59 +0800 Subject: [PATCH] Fix some issue to enable HEVC on iOS. - Add some missing classes. - Define OWT_USE_H265 when rtc_use_h265 is true. --- BUILD.gn | 2 ++ modules/rtp_rtcp/BUILD.gn | 2 ++ sdk/BUILD.gn | 23 +++++++++++---- .../Headers/WebRTC/RTCVideoCodecH265.h | 5 +--- .../RTCCodecSpecificInfoH265+Private.h | 25 +++++++++++++++++ .../video_codec/RTCCodecSpecificInfoH265.h | 28 +++++++++++++++++++ .../video_codec/RTCCodecSpecificInfoH265.mm | 28 +++++++++++++++++++ .../video_codec/RTCVideoCodecH265.mm | 10 +++---- .../video_codec/RTCVideoEncoderH265.mm | 18 ++++++------ .../components/video_codec/nalu_rewriter.cc | 6 ++++ .../components/video_codec/nalu_rewriter.h | 8 ++++++ .../native/src/objc_video_encoder_factory.mm | 8 ++++++ 12 files changed, 139 insertions(+), 24 deletions(-) create mode 100644 sdk/objc/components/video_codec/RTCCodecSpecificInfoH265+Private.h create mode 100644 sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.h create mode 100644 sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.mm diff --git a/BUILD.gn b/BUILD.gn index 68cd6c64d96..3276cce736f 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -206,6 +206,8 @@ config("common_inherited_config") { if (!rtc_use_h265) { defines += [ "DISABLE_H265" ] + } else { + defines += [ "OWT_USE_H265" ] } } diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index eec93dd947b..58db2f0c4b0 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -206,6 +206,8 @@ rtc_static_library("rtp_rtcp") { if (!rtc_use_h265) { defines += ["DISABLE_H265"] + } else { + defines += [ "OWT_USE_H265" ] } deps = [ diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index fe8bde3cbbc..fbac8619b3e 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -556,6 +556,16 @@ if (is_ios || is_mac) { "objc/components/video_codec/RTCH264ProfileLevelId.h", "objc/components/video_codec/RTCH264ProfileLevelId.mm", ] + if (rtc_use_h265) { + sources += [ + # TODO: Split this file. + "objc/Framework/Headers/WebRTC/RTCVideoCodecH265.h", + "objc/components/video_codec/RTCCodecSpecificInfoH265+Private.h", + "objc/components/video_codec/RTCCodecSpecificInfoH265.h", + "objc/components/video_codec/RTCCodecSpecificInfoH265.mm", + "objc/components/video_codec/RTCVideoCodecH265.mm", + ] + } if (is_ios) { sources += [ "objc/components/video_codec/UIDevice+H264Profile.h", @@ -1631,13 +1641,16 @@ if (is_ios || is_mac) { "objc/components/video_codec/RTCVideoEncoderFactoryH264.m", "objc/components/video_codec/RTCVideoEncoderH264.h", "objc/components/video_codec/RTCVideoEncoderH264.mm", - "objc/Framework/Headers/WebRTC/RTCVideoCodecH265.h", - "objc/components/video_codec/RTCVideoEncoderH265.mm", - "objc/components/video_codec/RTCVideoCodecH265.mm", - "objc/components/video_codec/RTCVideoDecoderH265.h", - "objc/components/video_codec/RTCVideoDecoderH265.mm", ] + if (rtc_use_h265) { + sources += [ + "objc/components/video_codec/RTCVideoDecoderH265.h", + "objc/components/video_codec/RTCVideoDecoderH265.mm", + "objc/components/video_codec/RTCVideoEncoderH265.mm", + ] + } + configs += [ "..:common_objc", ":used_from_extension", diff --git a/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH265.h b/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH265.h index 35157af596f..ac7557a394d 100644 --- a/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH265.h +++ b/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH265.h @@ -15,10 +15,7 @@ #import #import -RTC_OBJC_EXPORT -API_AVAILABLE(ios(11.0)) -@interface RTCCodecSpecificInfoH265 : NSObject -@end +RTC_OBJC_EXPORT extern NSString *const kRTCVideoCodecH265Name; /** Encoder. */ RTC_OBJC_EXPORT diff --git a/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265+Private.h b/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265+Private.h new file mode 100644 index 00000000000..a22d3594f55 --- /dev/null +++ b/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265+Private.h @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h */ + +#import "RTCCodecSpecificInfoH265.h" + +#include "modules/video_coding/include/video_codec_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +/* Interfaces for converting to/from internal C++ formats. */ +@interface RTCCodecSpecificInfoH265 () + +- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.h b/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.h new file mode 100644 index 00000000000..82865a9cdb4 --- /dev/null +++ b/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.h @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.h. */ + +#import + +#import "RTCCodecSpecificInfo.h" +#import "RTCMacros.h" + +/** Class for H265 specific config. */ +typedef NS_ENUM(NSUInteger, RTCH265PacketizationMode) { + RTCH265PacketizationModeNonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed + RTCH265PacketizationModeSingleNalUnit // Mode 0 - only single NALU allowed +}; + +RTC_OBJC_EXPORT +@interface RTCCodecSpecificInfoH265 : NSObject + +@property(nonatomic, assign) RTCH265PacketizationMode packetizationMode; + +@end diff --git a/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.mm b/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.mm new file mode 100644 index 00000000000..94a46985f72 --- /dev/null +++ b/sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.mm @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + /* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.mm */ + +#import "RTCCodecSpecificInfoH265+Private.h" + +// H265 specific settings. +@implementation RTCCodecSpecificInfoH265 + +@synthesize packetizationMode = _packetizationMode; + +- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo { + webrtc::CodecSpecificInfo codecSpecificInfo; + codecSpecificInfo.codecType = webrtc::kVideoCodecH265; + codecSpecificInfo.codecSpecific.H265.packetization_mode = + (webrtc::H265PacketizationMode)_packetizationMode; + + return codecSpecificInfo; +} + +@end diff --git a/sdk/objc/components/video_codec/RTCVideoCodecH265.mm b/sdk/objc/components/video_codec/RTCVideoCodecH265.mm index ac890e3e122..e2bb9866cb0 100644 --- a/sdk/objc/components/video_codec/RTCVideoCodecH265.mm +++ b/sdk/objc/components/video_codec/RTCVideoCodecH265.mm @@ -11,22 +11,20 @@ /* This file is borrowed from * webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH265.mm */ +#import "WebRTC/RTCVideoCodec.h" #import "WebRTC/RTCVideoCodecH265.h" #include - -#import "WebRTC/RTCVideoCodec.h" - +#include "media/base/media_constants.h" #include "rtc_base/time_utils.h" #include "system_wrappers/include/field_trial.h" +NSString *const kRTCVideoCodecH265Name = @(cricket::kH265CodecName); + static NSString* kH265CodecName = @"H265"; // TODO(jianjunz): This is value is not correct. static NSString* kLevel31Main = @"4d001f"; -@implementation RTCCodecSpecificInfoH265 -@end - // Encoder factory. @implementation RTCVideoEncoderFactoryH265 diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm b/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm index bbceb1ebb44..ed280b23aab 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm +++ b/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm @@ -9,17 +9,19 @@ * */ -#import "WebRTC/RTCVideoCodecH265.h" - #import -#include - -#if defined(WEBRTC_IOS) -#import "helpers/UIDevice+RTCDevice.h" -#endif +#import "RTCCodecSpecificInfoH265.h" #import "WebRTC/RTCVideoCodec.h" +#import "WebRTC/RTCVideoCodecH265.h" #import "WebRTC/RTCVideoFrame.h" #import "WebRTC/RTCVideoFrameBuffer.h" +#import "helpers.h" +#import "sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h" +#if defined(WEBRTC_IOS) +#import "helpers/UIDevice+RTCDevice.h" +#endif + +#include #include "common_video/h264/profile_level_id.h" #include "common_video/include/bitrate_adjuster.h" #include "libyuv/convert_from.h" @@ -28,8 +30,6 @@ #include "rtc_base/buffer.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" -#import "sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h" -#import "helpers.h" #include "sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" #include "system_wrappers/include/clock.h" diff --git a/sdk/objc/components/video_codec/nalu_rewriter.cc b/sdk/objc/components/video_codec/nalu_rewriter.cc index 9c6d18e79ef..29710034d7b 100644 --- a/sdk/objc/components/video_codec/nalu_rewriter.cc +++ b/sdk/objc/components/video_codec/nalu_rewriter.cc @@ -248,6 +248,7 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, return true; } +#ifdef OWT_USE_H265 bool H265CMSampleBufferToAnnexBBuffer( CMSampleBufferRef hvcc_sample_buffer, bool is_keyframe, @@ -474,6 +475,7 @@ bool H265AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, CFRelease(contiguous_buffer); return true; } +#endif CMVideoFormatDescriptionRef CreateVideoFormatDescription( const uint8_t* annexb_buffer, @@ -505,6 +507,7 @@ CMVideoFormatDescriptionRef CreateVideoFormatDescription( return description; } +#ifdef OWT_USE_H265 CMVideoFormatDescriptionRef CreateH265VideoFormatDescription( const uint8_t* annexb_buffer, size_t annexb_buffer_size) { @@ -539,6 +542,7 @@ CMVideoFormatDescriptionRef CreateH265VideoFormatDescription( } return description; } +#endif AnnexBBufferReader::AnnexBBufferReader(const uint8_t* annexb_buffer, size_t length) @@ -587,6 +591,7 @@ bool AnnexBBufferReader::SeekToNextNaluOfType(NaluType type) { return false; } +#ifdef OWT_USE_H265 bool AnnexBBufferReader::SeekToNextNaluOfType(H265::NaluType type) { for (; offset_ != offsets_.end(); ++offset_) { if (offset_->payload_size < 1) @@ -596,6 +601,7 @@ bool AnnexBBufferReader::SeekToNextNaluOfType(H265::NaluType type) { } return false; } +#endif AvccBufferWriter::AvccBufferWriter(uint8_t* const avcc_buffer, size_t length) : start_(avcc_buffer), offset_(0), length_(length) { diff --git a/sdk/objc/components/video_codec/nalu_rewriter.h b/sdk/objc/components/video_codec/nalu_rewriter.h index f0118a144c4..685855579c8 100644 --- a/sdk/objc/components/video_codec/nalu_rewriter.h +++ b/sdk/objc/components/video_codec/nalu_rewriter.h @@ -18,7 +18,9 @@ #include #include "common_video/h264/h264_common.h" +#ifdef OWT_USE_H265 #include "common_video/h265/h265_common.h" +#endif #include "modules/include/module_common_types.h" #include "rtc_base/buffer.h" @@ -48,6 +50,7 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, CMSampleBufferRef* out_sample_buffer, CMMemoryPoolRef memory_pool); +#ifdef OWT_USE_H265 // Converts a sample buffer emitted from the VideoToolbox encoder into a buffer // suitable for RTP. The sample buffer is in hvcc format whereas the rtp buffer // needs to be in Annex B format. Data is written directly to |annexb_buffer| @@ -70,6 +73,7 @@ bool H265AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, CMVideoFormatDescriptionRef video_format, CMSampleBufferRef* out_sample_buffer) __OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0); +#endif // Returns a video format description created from the sps/pps information in // the Annex B buffer. If there is no such information, nullptr is returned. @@ -78,10 +82,12 @@ CMVideoFormatDescriptionRef CreateVideoFormatDescription( const uint8_t* annexb_buffer, size_t annexb_buffer_size); +#ifdef OWT_USE_H265 CMVideoFormatDescriptionRef CreateH265VideoFormatDescription( const uint8_t* annexb_buffer, size_t annexb_buffer_size) __OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0); +#endif // Helper class for reading NALUs from an RTP Annex B buffer. class AnnexBBufferReader final { @@ -107,7 +113,9 @@ class AnnexBBufferReader final { // Return true if a NALU of the desired type is found, false if we // reached the end instead bool SeekToNextNaluOfType(H264::NaluType type); +#ifdef OWT_USE_H265 bool SeekToNextNaluOfType(H265::NaluType type); +#endif private: // Returns the the next offset that contains NALU data. diff --git a/sdk/objc/native/src/objc_video_encoder_factory.mm b/sdk/objc/native/src/objc_video_encoder_factory.mm index 989a402911f..33c03bba92e 100644 --- a/sdk/objc/native/src/objc_video_encoder_factory.mm +++ b/sdk/objc/native/src/objc_video_encoder_factory.mm @@ -15,6 +15,9 @@ #import "base/RTCVideoEncoder.h" #import "base/RTCVideoEncoderFactory.h" #import "components/video_codec/RTCCodecSpecificInfoH264+Private.h" +#ifdef OWT_USE_H265 +#import "components/video_codec/RTCCodecSpecificInfoH265+Private.h" +#endif #import "sdk/objc/api/peerconnection/RTCEncodedImage+Private.h" #import "sdk/objc/api/peerconnection/RTCRtpFragmentationHeader+Private.h" #import "sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h" @@ -62,6 +65,11 @@ int32_t RegisterEncodeCompleteCallback(EncodedImageCallback *callback) override if ([NSStringFromClass([info class]) isEqual:@"RTCCodecSpecificInfoH264"]) { // if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) { codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info nativeCodecSpecificInfo]; +#ifdef OWT_USE_H265 + } else if ([NSStringFromClass([info class]) isEqual:@"RTCCodecSpecificInfoH265"]) { + // if ([info isKindOfClass:[RTCCodecSpecificInfoH265 class]]) { + codecSpecificInfo = [(RTCCodecSpecificInfoH265 *)info nativeCodecSpecificInfo]; +#endif } std::unique_ptr fragmentationHeader =