From 61fc0e612d4261aabaae94481e6e485a7a9bde8c Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Wed, 12 Jun 2024 15:37:39 +0800 Subject: [PATCH] Update to m125. (#119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use M125 as the latest version and migrate historical patches to m125 Patches Group: ## 1. Update README.md https://github.com/webrtc-sdk/webrtc/pull/119/commits/b6c65fc7f0e81ed305e5578467b7198833665352 * Add Apache-2.0 license and some note to README.md. (#9) * Updated readme detailing changes from original (#42) * Adding membrane framework (#51) * Updated readme (#83) ## 2. Audio Device Optimization https://github.com/webrtc-sdk/webrtc/pull/119/commits/7454824ea6481cae2f70ad58fb1e1eb5d3ecdc88 * allow listen-only mode in AudioUnit, adjust when category changes (https://github.com/webrtc-sdk/webrtc/pull/2) * release mic when category changes (https://github.com/webrtc-sdk/webrtc/pull/5) * Change defaults to iOS defaults (https://github.com/webrtc-sdk/webrtc/pull/7) * Sync audio session config (https://github.com/webrtc-sdk/webrtc/pull/8) * feat: support bypass voice processing for iOS. (https://github.com/webrtc-sdk/webrtc/pull/15) * Remove MacBookPro audio pan right code (https://github.com/webrtc-sdk/webrtc/pull/22) * fix: Fix can't open mic alone when built-in AEC is enabled. (https://github.com/webrtc-sdk/webrtc/pull/29) * feat: add audio device changes detect for windows. (https://github.com/webrtc-sdk/webrtc/pull/41) * fix Linux compile (https://github.com/webrtc-sdk/webrtc/pull/47) * AudioUnit: Don't rely on category switch for mic indicator to turn off (https://github.com/webrtc-sdk/webrtc/pull/52) * Stop recording on mute (turn off mic indicator) (https://github.com/webrtc-sdk/webrtc/pull/55) * Cherry pick audio selection from m97 release (https://github.com/webrtc-sdk/webrtc/pull/35) * [Mac] Allow audio device selection (https://github.com/webrtc-sdk/webrtc/pull/21) * RTCAudioDeviceModule.outputDevice / inputDevice getter and setter (https://github.com/webrtc-sdk/webrtc/pull/80) * Allow custom audio processing by exposing AudioProcessingModule (https://github.com/webrtc-sdk/webrtc/pull/85) * Expose audio sample buffers for Android (https://github.com/webrtc-sdk/webrtc/pull/89) * feat: add external audio processor for android. (https://github.com/webrtc-sdk/webrtc/pull/103) * android: make audio output attributes modifiable (https://github.com/webrtc-sdk/webrtc/pull/118) * Fix external audio processor sample rate calculation (https://github.com/webrtc-sdk/webrtc/pull/108) * Expose remote audio sample buffers on RTCAudioTrack (https://github.com/webrtc-sdk/webrtc/pull/84) * Fix memory leak when creating audio CMSampleBuffer https://github.com/webrtc-sdk/webrtc/pull/86 ## 3. Simulcast/SVC support for iOS/Android. https://github.com/webrtc-sdk/webrtc/pull/119/commits/b0b9fe93b1b4f0f8cdd80919a05f999124ff8bce - Simulcast support for iOS SDK (#4) - Support for simulcast in Android SDK (#3) - include simulcast headers for mac also (#10) - Fix simulcast using hardware encoder on Android (#48) - Add scalabilityMode support for AV1/VP9. (#90) ## 4. Android improvements. https://github.com/webrtc-sdk/webrtc/pull/119/commits/9aaaab5a20fec2c84616ca709ac9c1aae6de9225 - Start/Stop receiving stream method for VideoTrack (#25) - Properly remove observer upon deconstruction (#26) - feat: Expose setCodecPreferences/getCapabilities for android. (#61) - fix: add WrappedVideoDecoderFactory.java. (#74) ## 5. Darwin improvements https://github.com/webrtc-sdk/webrtc/pull/119/commits/a13ea178e46ee2ab222f1553c9164b6556e1abd8 - [Mac/iOS] feat: Add RTCYUVHelper for darwin. (#28) - Cross-platform `RTCMTLVideoView` for both iOS / macOS (#40) - rotationOverride should not be assign (#44) - [ObjC] Expose properties / methods required for AV1 codec support (#60) - Workaround: Render PixelBuffer in RTCMTLVideoView (#58) - Improve iOS/macOS H264 encoder (#70) - fix: fix video encoder not resuming correctly upon foregrounding (#75). - add PrivacyInfo.xcprivacy to darwin frameworks. (#112) - Add NSPrivacyCollectedDataTypes key to xcprivacy file (#114) - Thread-safe `RTCInitFieldTrialDictionary` (#116) - Set RTCCameraVideoCapturer initial zoom factor (#121) - Unlock configuration before starting capture session (#122) ## 6. Desktop Capture for macOS. https://github.com/webrtc-sdk/webrtc/pull/119/commits/841d78f0631aff91590c1493909c4f3fccd8fafe - [Mac] feat: Support screen capture for macOS. (#24) (#36) - fix: Get thumbnails asynchronously. (#37) - fix: Use CVPixelBuffer to build DesktopCapture Frame, fix the crash caused by non-CVPixelBuffer frame in RTCVideoEncoderH264 that cannot be cropped. (#63) - Fix the crash when setting the fps of the virtual camera. (#62) ## 7. Frame Cryptor Support. https://github.com/webrtc-sdk/webrtc/pull/119/commits/fc08745d9c5218aed45829942a3590c97c19c76a - feat: Frame Cryptor (aes gcm/cbc). (#54) - feat: key ratchet/derive. (#66) - fix: skip invalid key when decryption failed. (#81) - Improve e2ee, add setSharedKey to KeyProvider. (#88) - add failure tolerance for framecryptor. (#91) - fix h264 freeze. (#93) - Fix/send frame cryptor events from signaling thread (#95) - more improvements for E2EE. (#96) - remove too verbose logs (#107) - Add key ring size to keyProviderOptions. (#109) ## 8. Other improvements. https://github.com/webrtc-sdk/webrtc/pull/119/commits/eed6c8a1bff6c0e16a850be07597e59ed8a68d7e - Added yuv_helper (#57) - ABGRToI420, ARGBToI420 & ARGBToRGB24 (#65) - more yuv wrappers (#87) - Fix naming for yuv helper (#113) - Fix missing `RTC_OBJC_TYPE` macros (#100) --------- Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Co-authored-by: David Zhao Co-authored-by: davidliu Co-authored-by: Angelika Serwa Co-authored-by: Théo Monnom # Conflicts: # README.md # media/engine/webrtc_video_engine.cc # media/engine/webrtc_video_engine.h # modules/audio_device/audio_device_impl.cc # sdk/BUILD.gn # sdk/android/BUILD.gn # sdk/android/api/org/webrtc/RtpParameters.java # sdk/android/api/org/webrtc/SimulcastVideoEncoder.java # sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java # sdk/android/api/org/webrtc/VideoCodecInfo.java # sdk/android/src/jni/pc/rtp_parameters.cc # sdk/android/src/jni/simulcast_video_encoder.cc # sdk/android/src/jni/simulcast_video_encoder.h # sdk/android/src/jni/video_codec_info.cc # sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h # sdk/objc/api/peerconnection/RTCAudioDeviceModule.h # sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm # sdk/objc/api/peerconnection/RTCAudioTrack.mm # sdk/objc/api/peerconnection/RTCIODevice+Private.h # sdk/objc/api/peerconnection/RTCIODevice.mm # sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h # sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm # sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h # sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm # sdk/objc/base/RTCAudioRenderer.h # sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h # sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm --- .gitignore | 6 + NOTICE | 26 + api/BUILD.gn | 1 + api/crypto/BUILD.gn | 18 + api/crypto/frame_crypto_transformer.cc | 727 ++++++++++++++++++ api/crypto/frame_crypto_transformer.h | 482 ++++++++++++ api/frame_transformer_interface.h | 2 + api/peer_connection_interface.h | 9 + api/video/BUILD.gn | 13 + api/video/yuv_helper.cc | 416 ++++++++++ api/video/yuv_helper.h | 316 ++++++++ api/video_codecs/video_encoder_factory.h | 7 +- audio/channel_receive.cc | 4 +- media/engine/webrtc_video_engine.cc | 19 +- media/engine/webrtc_video_engine.h | 1 - modules/audio_device/include/audio_device.h | 2 +- .../mac/screen_capturer_mac.mm | 11 +- ...sender_video_frame_transformer_delegate.cc | 2 + ...eam_receiver_frame_transformer_delegate.cc | 4 + pc/audio_rtp_receiver.cc | 2 +- pc/peer_connection.cc | 9 +- sdk/BUILD.gn | 96 ++- sdk/android/BUILD.gn | 39 + sdk/android/api/org/webrtc/AudioTrack.java | 48 ++ .../api/org/webrtc/AudioTrackSink.java | 27 + .../ExternalAudioProcessingFactory.java | 144 ++++ sdk/android/api/org/webrtc/FrameCryptor.java | 108 +++ .../api/org/webrtc/FrameCryptorAlgorithm.java | 22 + .../api/org/webrtc/FrameCryptorFactory.java | 44 ++ .../org/webrtc/FrameCryptorKeyProvider.java | 93 +++ .../webrtc/HardwareVideoEncoderFactory.java | 4 +- .../api/org/webrtc/LibaomAv1Encoder.java | 7 + .../api/org/webrtc/LibvpxVp9Encoder.java | 7 + .../api/org/webrtc/PeerConnection.java | 17 + .../api/org/webrtc/PeerConnectionFactory.java | 1 + sdk/android/api/org/webrtc/RtpParameters.java | 5 +- .../api/org/webrtc/SimulcastVideoEncoder.java | 7 +- .../api/org/webrtc/VideoCodecInfo.java | 20 +- .../webrtc/audio/JavaAudioDeviceModule.java | 20 +- ...ndroidVideoDecoderInstrumentationTest.java | 2 +- .../DefaultVideoEncoderFactoryTest.java | 4 +- .../SoftwareVideoDecoderFactoryTest.java | 3 +- .../SoftwareVideoEncoderFactoryTest.java | 3 +- .../org/webrtc/CodecsWrapperTestHelper.java | 3 +- .../src/java/org/webrtc/H264Utils.java | 5 +- .../webrtc/MediaCodecVideoDecoderFactory.java | 4 +- .../org/webrtc/audio/WebRtcAudioTrack.java | 23 +- sdk/android/src/jni/libaom_av1_encoder.cc | 12 + sdk/android/src/jni/pc/audio_sink.cc | 39 + sdk/android/src/jni/pc/audio_sink.h | 41 + sdk/android/src/jni/pc/audio_track.cc | 26 + .../pc/external_audio_processing_factory.cc | 143 ++++ .../pc/external_audio_processing_factory.h | 68 ++ .../pc/external_audio_processing_interface.h | 33 + .../src/jni/pc/external_audio_processor.cc | 72 ++ .../src/jni/pc/external_audio_processor.h | 57 ++ sdk/android/src/jni/pc/frame_cryptor.cc | 202 +++++ sdk/android/src/jni/pc/frame_cryptor.h | 49 ++ .../src/jni/pc/frame_cryptor_key_provider.cc | 123 +++ .../src/jni/pc/frame_cryptor_key_provider.h | 35 + sdk/android/src/jni/pc/peer_connection.cc | 3 + .../src/jni/pc/peer_connection_factory.cc | 1 + sdk/android/src/jni/pc/rtp_parameters.cc | 2 +- .../src/jni/simulcast_video_encoder.cc | 5 +- sdk/android/src/jni/simulcast_video_encoder.h | 2 +- sdk/android/src/jni/video_codec_info.cc | 38 +- sdk/android/src/jni/vp9_codec.cc | 16 + sdk/objc/PrivacyInfo.xcprivacy | 28 + .../api/RTCVideoRendererAdapter+Private.h | 2 +- sdk/objc/api/RTCVideoRendererAdapter.h | 4 +- sdk/objc/api/RTCVideoRendererAdapter.mm | 19 +- .../RTCAudioDeviceModule+Private.h | 2 +- .../peerconnection/RTCAudioDeviceModule.mm | 20 +- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 10 +- .../api/peerconnection/RTCConfiguration.h | 11 + .../api/peerconnection/RTCConfiguration.mm | 3 + .../peerconnection/RTCEncodedImage+Private.mm | 10 +- sdk/objc/api/peerconnection/RTCFieldTrials.mm | 31 +- .../peerconnection/RTCFrameCryptor+Private.h | 45 ++ sdk/objc/api/peerconnection/RTCFrameCryptor.h | 78 ++ .../api/peerconnection/RTCFrameCryptor.mm | 224 ++++++ .../RTCFrameCryptorKeyProvider+Private.h | 31 + .../RTCFrameCryptorKeyProvider.h | 62 ++ .../RTCFrameCryptorKeyProvider.mm | 124 +++ .../api/peerconnection/RTCIODevice+Private.h | 2 +- sdk/objc/api/peerconnection/RTCIODevice.mm | 2 +- .../peerconnection/RTCPeerConnectionFactory.h | 12 + .../RTCPeerConnectionFactory.mm | 48 +- ...nnectionFactoryBuilder+DefaultComponents.h | 4 +- ...nectionFactoryBuilder+DefaultComponents.mm | 6 +- .../RTCPeerConnectionFactoryBuilder.h | 4 +- .../RTCPeerConnectionFactoryBuilder.mm | 6 +- .../peerconnection/RTCRtpCodecParameters.h | 1 + .../peerconnection/RTCRtpCodecParameters.mm | 1 + .../peerconnection/RTCRtpEncodingParameters.h | 4 + .../RTCRtpEncodingParameters.mm | 7 + .../api/peerconnection/RTCRtpTransceiver.h | 4 + .../api/peerconnection/RTCRtpTransceiver.mm | 16 + .../RTCVideoCodecInfo+Private.mm | 17 +- sdk/objc/api/peerconnection/RTCVideoTrack.mm | 19 +- sdk/objc/api/video_codec/RTCVideoEncoderAV1.h | 2 + .../api/video_codec/RTCVideoEncoderAV1.mm | 14 + .../video_codec/RTCVideoEncoderSimulcast.h | 2 +- .../video_codec/RTCVideoEncoderSimulcast.mm | 58 +- sdk/objc/api/video_codec/RTCVideoEncoderVP9.h | 2 + .../api/video_codec/RTCVideoEncoderVP9.mm | 15 +- sdk/objc/base/RTCAudioRenderer.h | 6 +- sdk/objc/base/RTCVideoCodecInfo.h | 6 + sdk/objc/base/RTCVideoCodecInfo.m | 17 +- .../audio/RTCAudioSession+Configuration.mm | 27 + .../RTCNativeAudioSessionDelegateAdapter.h | 2 +- .../RTCNativeAudioSessionDelegateAdapter.mm | 2 +- .../capturer/RTCCameraVideoCapturer.h | 2 + .../capturer/RTCCameraVideoCapturer.m | 97 ++- .../capturer/RTCDesktopCapturer+Private.h | 49 ++ .../components/capturer/RTCDesktopCapturer.h | 61 ++ .../components/capturer/RTCDesktopCapturer.mm | 104 +++ .../capturer/RTCDesktopMediaList+Private.h | 40 + .../components/capturer/RTCDesktopMediaList.h | 51 ++ .../capturer/RTCDesktopMediaList.mm | 99 +++ .../capturer/RTCDesktopSource+Private.h | 37 + .../components/capturer/RTCDesktopSource.h | 40 + .../components/capturer/RTCDesktopSource.mm | 68 ++ .../network/RTCNetworkMonitor+Private.h | 14 +- .../components/network/RTCNetworkMonitor.h | 4 +- .../components/network/RTCNetworkMonitor.mm | 6 +- .../renderer/metal/RTCMTLI420Renderer.h | 2 +- .../renderer/metal/RTCMTLI420Renderer.mm | 2 +- .../renderer/metal/RTCMTLNSVideoView.m | 122 --- .../renderer/metal/RTCMTLNV12Renderer.h | 2 +- .../renderer/metal/RTCMTLNV12Renderer.mm | 2 +- .../renderer/metal/RTCMTLRGBRenderer.h | 3 +- .../renderer/metal/RTCMTLRGBRenderer.mm | 30 +- .../renderer/metal/RTCMTLRenderer+Private.h | 2 +- .../renderer/metal/RTCMTLRenderer.h | 4 +- .../renderer/metal/RTCMTLRenderer.mm | 2 +- .../renderer/metal/RTCMTLVideoView.h | 20 +- .../renderer/metal/RTCMTLVideoView.m | 99 ++- .../renderer/opengl/RTCDefaultShader.h | 2 +- .../renderer/opengl/RTCDefaultShader.mm | 2 +- .../renderer/opengl/RTCDisplayLinkTimer.h | 4 +- .../renderer/opengl/RTCDisplayLinkTimer.m | 10 +- .../renderer/opengl/RTCEAGLVideoView.m | 36 +- .../renderer/opengl/RTCI420TextureCache.h | 2 +- .../renderer/opengl/RTCI420TextureCache.mm | 2 +- .../renderer/opengl/RTCNSGLVideoView.m | 199 +++++ .../renderer/opengl/RTCNV12TextureCache.h | 2 +- .../renderer/opengl/RTCNV12TextureCache.m | 2 +- .../RTCDefaultVideoEncoderFactory.m | 6 +- .../RTCVideoEncoderFactorySimulcast.h | 2 +- .../RTCVideoEncoderFactorySimulcast.mm | 30 +- .../video_codec/RTCVideoEncoderH264.mm | 230 ++++-- sdk/objc/helpers/RTCYUVHelper.h | 118 +++ sdk/objc/helpers/RTCYUVHelper.mm | 179 +++++ sdk/objc/native/api/video_capturer.mm | 2 +- sdk/objc/native/src/audio/audio_device_ios.h | 4 +- sdk/objc/native/src/audio/audio_device_ios.mm | 2 +- sdk/objc/native/src/objc_audio_device.h | 4 +- sdk/objc/native/src/objc_audio_device.mm | 2 +- .../native/src/objc_audio_device_delegate.h | 2 +- .../native/src/objc_audio_device_delegate.mm | 2 +- sdk/objc/native/src/objc_desktop_capture.h | 70 ++ sdk/objc/native/src/objc_desktop_capture.mm | 205 +++++ sdk/objc/native/src/objc_desktop_media_list.h | 111 +++ .../native/src/objc_desktop_media_list.mm | 252 ++++++ sdk/objc/native/src/objc_network_monitor.h | 2 +- sdk/objc/native/src/objc_network_monitor.mm | 2 +- sdk/objc/native/src/objc_video_track_source.h | 6 +- .../native/src/objc_video_track_source.mm | 6 +- sdk/objc/unittests/RTCMTLVideoView_xctest.m | 6 +- .../RTCPeerConnectionFactoryBuilderTest.mm | 4 +- video/video_stream_encoder.cc | 12 +- 172 files changed, 6757 insertions(+), 535 deletions(-) create mode 100644 NOTICE create mode 100644 api/crypto/frame_crypto_transformer.cc create mode 100644 api/crypto/frame_crypto_transformer.h create mode 100644 api/video/yuv_helper.cc create mode 100644 api/video/yuv_helper.h create mode 100644 sdk/android/api/org/webrtc/AudioTrackSink.java create mode 100644 sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java create mode 100644 sdk/android/api/org/webrtc/FrameCryptor.java create mode 100644 sdk/android/api/org/webrtc/FrameCryptorAlgorithm.java create mode 100644 sdk/android/api/org/webrtc/FrameCryptorFactory.java create mode 100644 sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java create mode 100644 sdk/android/src/jni/pc/audio_sink.cc create mode 100644 sdk/android/src/jni/pc/audio_sink.h create mode 100644 sdk/android/src/jni/pc/external_audio_processing_factory.cc create mode 100644 sdk/android/src/jni/pc/external_audio_processing_factory.h create mode 100644 sdk/android/src/jni/pc/external_audio_processing_interface.h create mode 100644 sdk/android/src/jni/pc/external_audio_processor.cc create mode 100644 sdk/android/src/jni/pc/external_audio_processor.h create mode 100644 sdk/android/src/jni/pc/frame_cryptor.cc create mode 100644 sdk/android/src/jni/pc/frame_cryptor.h create mode 100644 sdk/android/src/jni/pc/frame_cryptor_key_provider.cc create mode 100644 sdk/android/src/jni/pc/frame_cryptor_key_provider.h create mode 100644 sdk/objc/PrivacyInfo.xcprivacy create mode 100644 sdk/objc/api/peerconnection/RTCFrameCryptor+Private.h create mode 100644 sdk/objc/api/peerconnection/RTCFrameCryptor.h create mode 100644 sdk/objc/api/peerconnection/RTCFrameCryptor.mm create mode 100644 sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider+Private.h create mode 100644 sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h create mode 100644 sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm create mode 100644 sdk/objc/components/capturer/RTCDesktopCapturer+Private.h create mode 100644 sdk/objc/components/capturer/RTCDesktopCapturer.h create mode 100644 sdk/objc/components/capturer/RTCDesktopCapturer.mm create mode 100644 sdk/objc/components/capturer/RTCDesktopMediaList+Private.h create mode 100644 sdk/objc/components/capturer/RTCDesktopMediaList.h create mode 100644 sdk/objc/components/capturer/RTCDesktopMediaList.mm create mode 100644 sdk/objc/components/capturer/RTCDesktopSource+Private.h create mode 100644 sdk/objc/components/capturer/RTCDesktopSource.h create mode 100644 sdk/objc/components/capturer/RTCDesktopSource.mm delete mode 100644 sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m create mode 100644 sdk/objc/components/renderer/opengl/RTCNSGLVideoView.m create mode 100644 sdk/objc/helpers/RTCYUVHelper.h create mode 100644 sdk/objc/helpers/RTCYUVHelper.mm create mode 100644 sdk/objc/native/src/objc_desktop_capture.h create mode 100644 sdk/objc/native/src/objc_desktop_capture.mm create mode 100644 sdk/objc/native/src/objc_desktop_media_list.h create mode 100644 sdk/objc/native/src/objc_desktop_media_list.mm diff --git a/.gitignore b/.gitignore index 8202b82014..2a4acfebb7 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,9 @@ /xcodebuild /.vscode !webrtc/* +/tmp.patch +/out-release +/out-debug +/node_modules +/libwebrtc +/args.txt diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000..3972578ec4 --- /dev/null +++ b/NOTICE @@ -0,0 +1,26 @@ +################################################################################### + +The following modifications follow Apache License 2.0 from shiguredo. + +https://github.com/webrtc-sdk/webrtc/commit/dfec53e93a0a1cb93f444caf50f844ec0068c7b7 +https://github.com/webrtc-sdk/webrtc/commit/403b4678543c5d4ac77bd1ea5753c02637b3bb89 +https://github.com/webrtc-sdk/webrtc/commit/77d5d685a90fb4bded17835ae72ec6671b26d696 + +Apache License 2.0 + +Copyright 2019-2021, Wandbox LLC (Original Author) +Copyright 2019-2021, Shiguredo Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +##################################################################################### \ No newline at end of file diff --git a/api/BUILD.gn b/api/BUILD.gn index c6ef19ac54..1731202a19 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -368,6 +368,7 @@ rtc_library("libjingle_peerconnection_api") { "video:encoded_image", "video:video_bitrate_allocator_factory", "video:video_frame", + "video:yuv_helper", "video:video_rtp_headers", "video_codecs:video_codecs_api", diff --git a/api/crypto/BUILD.gn b/api/crypto/BUILD.gn index 8d041ea059..9249b7716a 100644 --- a/api/crypto/BUILD.gn +++ b/api/crypto/BUILD.gn @@ -16,6 +16,24 @@ group("crypto") { ] } +rtc_library("frame_crypto_transformer") { + visibility = [ "*" ] + sources = [ + "frame_crypto_transformer.cc", + "frame_crypto_transformer.h", + ] + + deps = [ + "//api:frame_transformer_interface", + ] + + if (rtc_build_ssl) { + deps += [ "//third_party/boringssl" ] + } else { + configs += [ ":external_ssl_library" ] + } +} + rtc_library("options") { visibility = [ "*" ] sources = [ diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc new file mode 100644 index 0000000000..06ec888170 --- /dev/null +++ b/api/crypto/frame_crypto_transformer.cc @@ -0,0 +1,727 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "frame_crypto_transformer.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "api/array_view.h" +#include "common_video/h264/h264_common.h" +#include "modules/rtp_rtcp/source/rtp_format_h264.h" +#include "rtc_base/byte_buffer.h" +#include "rtc_base/logging.h" + +enum class EncryptOrDecrypt { kEncrypt = 0, kDecrypt }; + +#define Success 0 +#define ErrorUnexpected -1 +#define OperationError -2 +#define ErrorDataTooSmall -3 +#define ErrorInvalidAesGcmTagLength -4 + +webrtc::VideoCodecType get_video_codec_type( + webrtc::TransformableFrameInterface* frame) { + auto videoFrame = + static_cast(frame); + return videoFrame->header().codec; +} + +webrtc::H264PacketizationMode get_h264_packetization_mode( + webrtc::TransformableFrameInterface* frame) { + auto video_frame = + static_cast(frame); + const auto& h264_header = absl::get( + video_frame->header().video_type_header); + return h264_header.packetization_mode; +} + +const EVP_AEAD* GetAesGcmAlgorithmFromKeySize(size_t key_size_bytes) { + switch (key_size_bytes) { + case 16: + return EVP_aead_aes_128_gcm(); + case 32: + return EVP_aead_aes_256_gcm(); + default: + return nullptr; + } +} + +const EVP_CIPHER* GetAesCbcAlgorithmFromKeySize(size_t key_size_bytes) { + switch (key_size_bytes) { + case 16: + return EVP_aes_128_cbc(); + case 32: + return EVP_aes_256_cbc(); + default: + return nullptr; + } +} + +inline bool FrameIsH264(webrtc::TransformableFrameInterface* frame, + webrtc::FrameCryptorTransformer::MediaType type) { + switch (type) { + case webrtc::FrameCryptorTransformer::MediaType::kVideoFrame: { + auto videoFrame = + static_cast(frame); + return videoFrame->header().codec == + webrtc::VideoCodecType::kVideoCodecH264; + } + default: + return false; + } +} + +inline bool NeedsRbspUnescaping(const uint8_t* frameData, size_t frameSize) { + for (size_t i = 0; i < frameSize - 3; ++i) { + if (frameData[i] == 0 && frameData[i + 1] == 0 && frameData[i + 2] == 3) + return true; + } + return false; +} + +std::string to_uint8_list(const uint8_t* data, int len) { + std::stringstream ss; + ss << "["; + for (int i = 0; i < len; i++) { + ss << static_cast(data[i]) << ","; + } + ss << "]"; + return ss.str(); +} + +std::string to_hex(const uint8_t* data, int len) { + std::stringstream ss; + ss << std::uppercase << std::hex << std::setfill('0'); + for (int i = 0; i < len; i++) { + ss << std::setw(2) << static_cast(data[i]); + } + return ss.str(); +} + +uint8_t get_unencrypted_bytes(webrtc::TransformableFrameInterface* frame, + webrtc::FrameCryptorTransformer::MediaType type) { + uint8_t unencrypted_bytes = 0; + switch (type) { + case webrtc::FrameCryptorTransformer::MediaType::kAudioFrame: + unencrypted_bytes = 1; + break; + case webrtc::FrameCryptorTransformer::MediaType::kVideoFrame: { + auto videoFrame = + static_cast(frame); + if (videoFrame->header().codec == + webrtc::VideoCodecType::kVideoCodecAV1) { + unencrypted_bytes = 0; + } else if (videoFrame->header().codec == + webrtc::VideoCodecType::kVideoCodecVP8) { + unencrypted_bytes = videoFrame->IsKeyFrame() ? 10 : 3; + } else if (videoFrame->header().codec == + webrtc::VideoCodecType::kVideoCodecH264) { + rtc::ArrayView date_in = frame->GetData(); + std::vector nalu_indices = + webrtc::H264::FindNaluIndices(date_in.data(), date_in.size()); + + int idx = 0; + for (const auto& index : nalu_indices) { + const uint8_t* slice = date_in.data() + index.payload_start_offset; + webrtc::H264::NaluType nalu_type = + webrtc::H264::ParseNaluType(slice[0]); + switch (nalu_type) { + case webrtc::H264::NaluType::kIdr: + case webrtc::H264::NaluType::kSlice: + unencrypted_bytes = index.payload_start_offset + 2; + RTC_LOG(LS_INFO) + << "NonParameterSetNalu::payload_size: " << index.payload_size + << ", nalu_type " << nalu_type << ", NaluIndex [" << idx++ + << "] offset: " << index.payload_start_offset; + return unencrypted_bytes; + default: + break; + } + } + } + break; + } + default: + break; + } + return unencrypted_bytes; +} + +int DerivePBKDF2KeyFromRawKey(const std::vector raw_key, + const std::vector& salt, + unsigned int optional_length_bits, + std::vector* derived_key) { + size_t key_size_bytes = optional_length_bits / 8; + derived_key->resize(key_size_bytes); + + if (PKCS5_PBKDF2_HMAC((const char*)raw_key.data(), raw_key.size(), + salt.data(), salt.size(), 100000, EVP_sha256(), + key_size_bytes, derived_key->data()) != 1) { + RTC_LOG(LS_ERROR) << "Failed to derive AES key from password."; + return ErrorUnexpected; + } + + RTC_LOG(LS_INFO) << "raw_key " + << to_uint8_list(raw_key.data(), raw_key.size()) << " len " + << raw_key.size() << " slat << " + << to_uint8_list(salt.data(), salt.size()) << " len " + << salt.size() << "\n derived_key " + << to_uint8_list(derived_key->data(), derived_key->size()) + << " len " << derived_key->size(); + + return Success; +} + +int AesGcmEncryptDecrypt(EncryptOrDecrypt mode, + const std::vector raw_key, + const rtc::ArrayView data, + unsigned int tag_length_bytes, + rtc::ArrayView iv, + rtc::ArrayView additional_data, + const EVP_AEAD* aead_alg, + std::vector* buffer) { + bssl::ScopedEVP_AEAD_CTX ctx; + + if (!aead_alg) { + RTC_LOG(LS_ERROR) << "Invalid AES-GCM key size."; + return ErrorUnexpected; + } + + if (!EVP_AEAD_CTX_init(ctx.get(), aead_alg, raw_key.data(), raw_key.size(), + tag_length_bytes, nullptr)) { + RTC_LOG(LS_ERROR) << "Failed to initialize AES-GCM context."; + return OperationError; + } + + size_t len; + int ok; + + if (mode == EncryptOrDecrypt::kDecrypt) { + if (data.size() < tag_length_bytes) { + RTC_LOG(LS_ERROR) << "Data too small for AES-GCM tag."; + return ErrorDataTooSmall; + } + + buffer->resize(data.size() - tag_length_bytes); + + ok = EVP_AEAD_CTX_open(ctx.get(), buffer->data(), &len, buffer->size(), + iv.data(), iv.size(), data.data(), data.size(), + additional_data.data(), additional_data.size()); + } else { + buffer->resize(data.size() + EVP_AEAD_max_overhead(aead_alg)); + + ok = EVP_AEAD_CTX_seal(ctx.get(), buffer->data(), &len, buffer->size(), + iv.data(), iv.size(), data.data(), data.size(), + additional_data.data(), additional_data.size()); + } + + if (!ok) { + RTC_LOG(LS_WARNING) << "Failed to perform AES-GCM operation."; + return OperationError; + } + + buffer->resize(len); + + return Success; +} + +int AesCbcEncryptDecrypt(EncryptOrDecrypt mode, + const std::vector& raw_key, + rtc::ArrayView iv, + const rtc::ArrayView input, + std::vector* output) { + const EVP_CIPHER* cipher = GetAesCbcAlgorithmFromKeySize(raw_key.size()); + RTC_DCHECK(cipher); // Already handled in Init(); + RTC_DCHECK_EQ(EVP_CIPHER_iv_length(cipher), iv.size()); + RTC_DCHECK_EQ(EVP_CIPHER_key_length(cipher), raw_key.size()); + + bssl::ScopedEVP_CIPHER_CTX ctx; + if (!EVP_CipherInit_ex(ctx.get(), cipher, nullptr, + reinterpret_cast(raw_key.data()), + iv.data(), + mode == EncryptOrDecrypt::kEncrypt ? 1 : 0)) { + return OperationError; + } + + // Encrypting needs a block size of space to allow for any padding. + output->resize(input.size() + + (mode == EncryptOrDecrypt::kEncrypt ? iv.size() : 0)); + int out_len; + if (!EVP_CipherUpdate(ctx.get(), output->data(), &out_len, input.data(), + input.size())) + return OperationError; + + // Write out the final block plus padding (if any) to the end of the data + // just written. + int tail_len; + if (!EVP_CipherFinal_ex(ctx.get(), output->data() + out_len, &tail_len)) + return OperationError; + + out_len += tail_len; + RTC_CHECK_LE(out_len, static_cast(output->size())); + return Success; +} + +int AesEncryptDecrypt(EncryptOrDecrypt mode, + webrtc::FrameCryptorTransformer::Algorithm algorithm, + const std::vector& raw_key, + rtc::ArrayView iv, + rtc::ArrayView additional_data, + const rtc::ArrayView data, + std::vector* buffer) { + switch (algorithm) { + case webrtc::FrameCryptorTransformer::Algorithm::kAesGcm: { + unsigned int tag_length_bits = 128; + return AesGcmEncryptDecrypt( + mode, raw_key, data, tag_length_bits / 8, iv, additional_data, + GetAesGcmAlgorithmFromKeySize(raw_key.size()), buffer); + } + case webrtc::FrameCryptorTransformer::Algorithm::kAesCbc: + return AesCbcEncryptDecrypt(mode, raw_key, iv, data, buffer); + } +} +namespace webrtc { + +FrameCryptorTransformer::FrameCryptorTransformer( + rtc::Thread* signaling_thread, + const std::string participant_id, + MediaType type, + Algorithm algorithm, + rtc::scoped_refptr key_provider) + : signaling_thread_(signaling_thread), + thread_(rtc::Thread::Create()), + participant_id_(participant_id), + type_(type), + algorithm_(algorithm), + key_provider_(key_provider) { + RTC_DCHECK(key_provider_ != nullptr); + thread_->SetName("FrameCryptorTransformer", this); + thread_->Start(); +} + +FrameCryptorTransformer::~FrameCryptorTransformer() { + thread_->Stop(); +} + +void FrameCryptorTransformer::Transform( + std::unique_ptr frame) { + webrtc::MutexLock lock(&sink_mutex_); + if (sink_callback_ == nullptr && sink_callbacks_.size() == 0) { + RTC_LOG(LS_WARNING) + << "FrameCryptorTransformer::Transform sink_callback_ is NULL"; + return; + } + + // do encrypt or decrypt here... + switch (frame->GetDirection()) { + case webrtc::TransformableFrameInterface::Direction::kSender: + RTC_DCHECK(thread_ != nullptr); + thread_->PostTask([frame = std::move(frame), this]() mutable { + encryptFrame(std::move(frame)); + }); + break; + case webrtc::TransformableFrameInterface::Direction::kReceiver: + RTC_DCHECK(thread_ != nullptr); + thread_->PostTask([frame = std::move(frame), this]() mutable { + decryptFrame(std::move(frame)); + }); + break; + case webrtc::TransformableFrameInterface::Direction::kUnknown: + // do nothing + RTC_LOG(LS_INFO) << "FrameCryptorTransformer::Transform() kUnknown"; + break; + } +} + +void FrameCryptorTransformer::encryptFrame( + std::unique_ptr frame) { + bool enabled_cryption = false; + rtc::scoped_refptr sink_callback = nullptr; + { + webrtc::MutexLock lock(&mutex_); + enabled_cryption = enabled_cryption_; + if (type_ == webrtc::FrameCryptorTransformer::MediaType::kAudioFrame) { + sink_callback = sink_callback_; + } else { + sink_callback = sink_callbacks_[frame->GetSsrc()]; + } + } + + if (sink_callback == nullptr) { + RTC_LOG(LS_WARNING) + << "FrameCryptorTransformer::encryptFrame() sink_callback is NULL"; + if (last_enc_error_ != FrameCryptionState::kInternalError) { + last_enc_error_ = FrameCryptionState::kInternalError; + onFrameCryptionStateChanged(last_enc_error_); + } + return; + } + + rtc::ArrayView date_in = frame->GetData(); + if (date_in.size() == 0 || !enabled_cryption) { + RTC_LOG(LS_WARNING) << "FrameCryptorTransformer::encryptFrame() " + "date_in.size() == 0 || enabled_cryption == false"; + if(key_provider_->options().discard_frame_when_cryptor_not_ready) { + return; + } + sink_callback->OnTransformedFrame(std::move(frame)); + return; + } + + auto key_handler = key_provider_->options().shared_key + ? key_provider_->GetSharedKey(participant_id_) + : key_provider_->GetKey(participant_id_); + + if (key_handler == nullptr || key_handler->GetKeySet(key_index_) == nullptr) { + RTC_LOG(LS_INFO) << "FrameCryptorTransformer::encryptFrame() no keys, or " + "key_index[" + << key_index_ << "] out of range for participant " + << participant_id_; + if (last_enc_error_ != FrameCryptionState::kMissingKey) { + last_enc_error_ = FrameCryptionState::kMissingKey; + onFrameCryptionStateChanged(last_enc_error_); + } + return; + } + + auto key_set = key_handler->GetKeySet(key_index_); + uint8_t unencrypted_bytes = get_unencrypted_bytes(frame.get(), type_); + + rtc::Buffer frame_header(unencrypted_bytes); + for (size_t i = 0; i < unencrypted_bytes; i++) { + frame_header[i] = date_in[i]; + } + + rtc::Buffer frame_trailer(2); + frame_trailer[0] = getIvSize(); + frame_trailer[1] = key_index_; + rtc::Buffer iv = makeIv(frame->GetSsrc(), frame->GetTimestamp()); + + rtc::Buffer payload(date_in.size() - unencrypted_bytes); + for (size_t i = unencrypted_bytes; i < date_in.size(); i++) { + payload[i - unencrypted_bytes] = date_in[i]; + } + + std::vector buffer; + if (AesEncryptDecrypt(EncryptOrDecrypt::kEncrypt, algorithm_, + key_set->encryption_key, iv, frame_header, payload, + &buffer) == Success) { + rtc::Buffer encrypted_payload(buffer.data(), buffer.size()); + rtc::Buffer tag(encrypted_payload.data() + encrypted_payload.size() - 16, + 16); + rtc::Buffer data_without_header; + data_without_header.AppendData(encrypted_payload); + data_without_header.AppendData(iv); + data_without_header.AppendData(frame_trailer); + + rtc::Buffer data_out; + data_out.AppendData(frame_header); + + if (FrameIsH264(frame.get(), type_)) { + H264::WriteRbsp(data_without_header.data(), data_without_header.size(), + &data_out); + } else { + data_out.AppendData(data_without_header); + RTC_CHECK_EQ(data_out.size(), frame_header.size() + + encrypted_payload.size() + iv.size() + + frame_trailer.size()); + } + + frame->SetData(data_out); + + if (last_enc_error_ != FrameCryptionState::kOk) { + last_enc_error_ = FrameCryptionState::kOk; + onFrameCryptionStateChanged(last_enc_error_); + } + sink_callback->OnTransformedFrame(std::move(frame)); + } else { + if (last_enc_error_ != FrameCryptionState::kEncryptionFailed) { + last_enc_error_ = FrameCryptionState::kEncryptionFailed; + onFrameCryptionStateChanged(last_enc_error_); + } + RTC_LOG(LS_ERROR) << "FrameCryptorTransformer::encryptFrame() failed"; + } +} + +void FrameCryptorTransformer::decryptFrame( + std::unique_ptr frame) { + bool enabled_cryption = false; + rtc::scoped_refptr sink_callback = nullptr; + { + webrtc::MutexLock lock(&mutex_); + enabled_cryption = enabled_cryption_; + if (type_ == webrtc::FrameCryptorTransformer::MediaType::kAudioFrame) { + sink_callback = sink_callback_; + } else { + sink_callback = sink_callbacks_[frame->GetSsrc()]; + } + } + + if (sink_callback == nullptr) { + RTC_LOG(LS_WARNING) + << "FrameCryptorTransformer::decryptFrame() sink_callback is NULL"; + if (last_dec_error_ != FrameCryptionState::kInternalError) { + last_dec_error_ = FrameCryptionState::kInternalError; + onFrameCryptionStateChanged(last_dec_error_); + } + return; + } + + rtc::ArrayView date_in = frame->GetData(); + + if (date_in.size() == 0 || !enabled_cryption) { + RTC_LOG(LS_WARNING) << "FrameCryptorTransformer::decryptFrame() " + "date_in.size() == 0 || enabled_cryption == false"; + if(key_provider_->options().discard_frame_when_cryptor_not_ready) { + return; + } + + sink_callback->OnTransformedFrame(std::move(frame)); + return; + } + + auto uncrypted_magic_bytes = key_provider_->options().uncrypted_magic_bytes; + if (uncrypted_magic_bytes.size() > 0 && + date_in.size() >= uncrypted_magic_bytes.size()) { + auto tmp = date_in.subview(date_in.size() - (uncrypted_magic_bytes.size()), + uncrypted_magic_bytes.size()); + auto data = std::vector(tmp.begin(), tmp.end()); + if (uncrypted_magic_bytes == data) { + RTC_CHECK_EQ(tmp.size(), uncrypted_magic_bytes.size()); + RTC_LOG(LS_INFO) << "FrameCryptorTransformer::uncrypted_magic_bytes( tmp " + << to_hex(tmp.data(), tmp.size()) << ", magic bytes " + << to_hex(uncrypted_magic_bytes.data(), + uncrypted_magic_bytes.size()) + << ")"; + + // magic bytes detected, this is a non-encrypted frame, skip frame + // decryption. + rtc::Buffer data_out; + data_out.AppendData( + date_in.subview(0, date_in.size() - uncrypted_magic_bytes.size())); + frame->SetData(data_out); + sink_callback->OnTransformedFrame(std::move(frame)); + return; + } + } + + uint8_t unencrypted_bytes = get_unencrypted_bytes(frame.get(), type_); + + rtc::Buffer frame_header(unencrypted_bytes); + for (size_t i = 0; i < unencrypted_bytes; i++) { + frame_header[i] = date_in[i]; + } + + rtc::Buffer frame_trailer(2); + frame_trailer[0] = date_in[date_in.size() - 2]; + frame_trailer[1] = date_in[date_in.size() - 1]; + uint8_t ivLength = frame_trailer[0]; + uint8_t key_index = frame_trailer[1]; + + if (ivLength != getIvSize()) { + RTC_LOG(LS_WARNING) << "FrameCryptorTransformer::decryptFrame() ivLength[" + << static_cast(ivLength) << "] != getIvSize()[" + << static_cast(getIvSize()) << "]"; + if (last_dec_error_ != FrameCryptionState::kDecryptionFailed) { + last_dec_error_ = FrameCryptionState::kDecryptionFailed; + onFrameCryptionStateChanged(last_dec_error_); + } + return; + } + + auto key_handler = key_provider_->options().shared_key + ? key_provider_->GetSharedKey(participant_id_) + : key_provider_->GetKey(participant_id_); + + if (0 > key_index || key_index >= key_provider_->options().key_ring_size || key_handler == nullptr || + key_handler->GetKeySet(key_index) == nullptr) { + RTC_LOG(LS_INFO) << "FrameCryptorTransformer::decryptFrame() no keys, or " + "key_index[" + << key_index << "] out of range for participant " + << participant_id_; + if (last_dec_error_ != FrameCryptionState::kMissingKey) { + last_dec_error_ = FrameCryptionState::kMissingKey; + onFrameCryptionStateChanged(last_dec_error_); + } + return; + } + + if (last_dec_error_ == kDecryptionFailed && !key_handler->HasValidKey()) { + // if decryption failed and we have an invalid key, + // please try to decrypt with the next new key + return; + } + + auto key_set = key_handler->GetKeySet(key_index); + + rtc::Buffer iv = rtc::Buffer(ivLength); + for (size_t i = 0; i < ivLength; i++) { + iv[i] = date_in[date_in.size() - 2 - ivLength + i]; + } + + rtc::Buffer encrypted_buffer(date_in.size() - unencrypted_bytes); + for (size_t i = unencrypted_bytes; i < date_in.size(); i++) { + encrypted_buffer[i - unencrypted_bytes] = date_in[i]; + } + + if (FrameIsH264(frame.get(), type_) && + NeedsRbspUnescaping(encrypted_buffer.data(), encrypted_buffer.size())) { + encrypted_buffer.SetData( + H264::ParseRbsp(encrypted_buffer.data(), encrypted_buffer.size())); + } + + rtc::Buffer encrypted_payload(encrypted_buffer.size() - ivLength - 2); + for (size_t i = 0; i < encrypted_payload.size(); i++) { + encrypted_payload[i] = encrypted_buffer[i]; + } + + rtc::Buffer tag(encrypted_payload.data() + encrypted_payload.size() - 16, 16); + std::vector buffer; + + int ratchet_count = 0; + auto initialKeyMaterial = key_set->material; + bool decryption_success = false; + if (AesEncryptDecrypt(EncryptOrDecrypt::kDecrypt, algorithm_, + key_set->encryption_key, iv, frame_header, + encrypted_payload, &buffer) == Success) { + decryption_success = true; + } else { + RTC_LOG(LS_WARNING) << "FrameCryptorTransformer::decryptFrame() failed"; + rtc::scoped_refptr ratcheted_key_set; + auto currentKeyMaterial = key_set->material; + if (key_provider_->options().ratchet_window_size > 0) { + while (ratchet_count < key_provider_->options().ratchet_window_size) { + ratchet_count++; + + RTC_LOG(LS_INFO) << "ratcheting key attempt " << ratchet_count << " of " + << key_provider_->options().ratchet_window_size; + + auto new_material = key_handler->RatchetKeyMaterial(currentKeyMaterial); + ratcheted_key_set = key_handler->DeriveKeys( + new_material, key_provider_->options().ratchet_salt, 128); + + if (AesEncryptDecrypt(EncryptOrDecrypt::kDecrypt, algorithm_, + ratcheted_key_set->encryption_key, iv, + frame_header, encrypted_payload, + &buffer) == Success) { + RTC_LOG(LS_INFO) << "FrameCryptorTransformer::decryptFrame() " + "ratcheted to key_index=" + << static_cast(key_index); + decryption_success = true; + // success, so we set the new key + key_handler->SetKeyFromMaterial(new_material, key_index); + key_handler->SetHasValidKey(); + if (last_dec_error_ != FrameCryptionState::kKeyRatcheted) { + last_dec_error_ = FrameCryptionState::kKeyRatcheted; + onFrameCryptionStateChanged(last_dec_error_); + } + break; + } + // for the next ratchet attempt + currentKeyMaterial = new_material; + } + + /* Since the key it is first send and only afterwards actually used for + encrypting, there were situations when the decrypting failed due to the + fact that the received frame was not encrypted yet and ratcheting, of + course, did not solve the problem. So if we fail RATCHET_WINDOW_SIZE + times, we come back to the initial key. + */ + if (!decryption_success || + ratchet_count >= key_provider_->options().ratchet_window_size) { + key_handler->SetKeyFromMaterial(initialKeyMaterial, key_index); + } + } + } + + if (!decryption_success) { + if (key_handler->DecryptionFailure()) { + if (last_dec_error_ != FrameCryptionState::kDecryptionFailed) { + last_dec_error_ = FrameCryptionState::kDecryptionFailed; + onFrameCryptionStateChanged(last_dec_error_); + } + } + return; + } + + rtc::Buffer payload(buffer.data(), buffer.size()); + rtc::Buffer data_out; + data_out.AppendData(frame_header); + data_out.AppendData(payload); + frame->SetData(data_out); + + if (last_dec_error_ != FrameCryptionState::kOk) { + last_dec_error_ = FrameCryptionState::kOk; + onFrameCryptionStateChanged(last_dec_error_); + } + sink_callback->OnTransformedFrame(std::move(frame)); +} + +void FrameCryptorTransformer::onFrameCryptionStateChanged( + FrameCryptionState state) { + webrtc::MutexLock lock(&mutex_); + if (observer_) { + RTC_DCHECK(signaling_thread_ != nullptr); + signaling_thread_->PostTask([observer = observer_, state = state, + participant_id = participant_id_]() mutable { + observer->OnFrameCryptionStateChanged(participant_id, state); + }); + } +} + +rtc::Buffer FrameCryptorTransformer::makeIv(uint32_t ssrc, uint32_t timestamp) { + uint32_t send_count = 0; + if (send_counts_.find(ssrc) == send_counts_.end()) { + srand((unsigned)time(NULL)); + send_counts_[ssrc] = floor(rand() * 0xFFFF); + } else { + send_count = send_counts_[ssrc]; + } + rtc::ByteBufferWriter buf; + buf.WriteUInt32(ssrc); + buf.WriteUInt32(timestamp); + buf.WriteUInt32(timestamp - (send_count % 0xFFFF)); + send_counts_[ssrc] = send_count + 1; + + RTC_CHECK_EQ(buf.Length(), getIvSize()); + + return rtc::Buffer(buf.Data(), buf.Length()); +} + +uint8_t FrameCryptorTransformer::getIvSize() { + switch (algorithm_) { + case Algorithm::kAesGcm: + return 12; + case Algorithm::kAesCbc: + return 16; + default: + return 0; + } +} + +} // namespace webrtc diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h new file mode 100644 index 0000000000..9689ec1593 --- /dev/null +++ b/api/crypto/frame_crypto_transformer.h @@ -0,0 +1,482 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_FRAME_CRYPTOR_TRANSFORMER_H_ +#define WEBRTC_FRAME_CRYPTOR_TRANSFORMER_H_ + +#include + +#include "api/frame_transformer_interface.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/task_queue/task_queue_base.h" +#include "rtc_base/buffer.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/rtc_export.h" +#include "rtc_base/thread.h" + +int DerivePBKDF2KeyFromRawKey(const std::vector raw_key, + const std::vector& salt, + unsigned int optional_length_bits, + std::vector* derived_key); + +namespace webrtc { + +const size_t DEFAULT_KEYRING_SIZE = 16; +const size_t MAX_KEYRING_SIZE = 255; + +class ParticipantKeyHandler; + +struct KeyProviderOptions { + bool shared_key; + std::vector ratchet_salt; + std::vector uncrypted_magic_bytes; + int ratchet_window_size; + int failure_tolerance; + // key ring size should be between 1 and 255 + int key_ring_size; + bool discard_frame_when_cryptor_not_ready; + KeyProviderOptions() + : shared_key(false), + ratchet_window_size(0), + failure_tolerance(-1), + key_ring_size(DEFAULT_KEYRING_SIZE), + discard_frame_when_cryptor_not_ready(false) {} + KeyProviderOptions(KeyProviderOptions& copy) + : shared_key(copy.shared_key), + ratchet_salt(copy.ratchet_salt), + uncrypted_magic_bytes(copy.uncrypted_magic_bytes), + ratchet_window_size(copy.ratchet_window_size), + failure_tolerance(copy.failure_tolerance), + key_ring_size(copy.key_ring_size) {} +}; + +class KeyProvider : public rtc::RefCountInterface { + public: + virtual bool SetSharedKey(int key_index, std::vector key) = 0; + + virtual const rtc::scoped_refptr GetSharedKey( + const std::string participant_id) = 0; + + virtual const std::vector RatchetSharedKey(int key_index) = 0; + + virtual const std::vector ExportSharedKey(int key_index) const = 0; + + virtual bool SetKey(const std::string participant_id, + int key_index, + std::vector key) = 0; + + virtual const rtc::scoped_refptr GetKey( + const std::string participant_id) const = 0; + + virtual const std::vector RatchetKey( + const std::string participant_id, + int key_index) = 0; + + virtual const std::vector ExportKey(const std::string participant_id, + int key_index) const = 0; + + virtual void SetSifTrailer(const std::vector trailer) = 0; + + virtual KeyProviderOptions& options() = 0; + + protected: + virtual ~KeyProvider() {} +}; + +class ParticipantKeyHandler : public rtc::RefCountInterface { + public: + struct KeySet : public rtc::RefCountInterface { + std::vector material; + std::vector encryption_key; + KeySet(std::vector material, std::vector encryptionKey) + : material(material), encryption_key(encryptionKey) {} + }; + + public: + ParticipantKeyHandler(KeyProvider* key_provider) + : key_provider_(key_provider) { + int key_ring_size = key_provider_->options().key_ring_size; + if(key_ring_size <= 0) { + key_ring_size = DEFAULT_KEYRING_SIZE; + } else if (key_ring_size > (int)MAX_KEYRING_SIZE) { + // Keyring size needs to be between 1 and 256 + key_ring_size = MAX_KEYRING_SIZE; + } + crypto_key_ring_.resize(key_ring_size); + } + + virtual ~ParticipantKeyHandler() = default; + + rtc::scoped_refptr Clone() { + auto clone = rtc::make_ref_counted(key_provider_); + clone->crypto_key_ring_ = crypto_key_ring_; + clone->current_key_index_ = current_key_index_; + clone->has_valid_key_ = has_valid_key_; + return clone; + } + + virtual std::vector RatchetKey(int key_index) { + auto key_set = GetKeySet(key_index); + if (!key_set) { + return std::vector(); + } + auto current_material = key_set->material; + std::vector new_material; + if (DerivePBKDF2KeyFromRawKey(current_material, + key_provider_->options().ratchet_salt, 256, + &new_material) != 0) { + return std::vector(); + } + SetKeyFromMaterial(new_material, + key_index != -1 ? key_index : current_key_index_); + SetHasValidKey(); + return new_material; + } + + virtual rtc::scoped_refptr GetKeySet(int key_index) { + webrtc::MutexLock lock(&mutex_); + return crypto_key_ring_[key_index != -1 ? key_index : current_key_index_]; + } + + virtual void SetKey(std::vector password, int key_index) { + SetKeyFromMaterial(password, key_index); + SetHasValidKey(); + } + + std::vector RatchetKeyMaterial( + std::vector current_material) { + std::vector new_material; + if (DerivePBKDF2KeyFromRawKey(current_material, + key_provider_->options().ratchet_salt, 256, + &new_material) != 0) { + return std::vector(); + } + return new_material; + } + + rtc::scoped_refptr DeriveKeys(std::vector password, + std::vector ratchet_salt, + unsigned int optional_length_bits) { + std::vector derived_key; + if (DerivePBKDF2KeyFromRawKey(password, ratchet_salt, optional_length_bits, + &derived_key) == 0) { + return rtc::make_ref_counted(password, derived_key); + } + return nullptr; + } + + bool HasValidKey() { + webrtc::MutexLock lock(&mutex_); + return has_valid_key_; + } + + void SetHasValidKey() { + webrtc::MutexLock lock(&mutex_); + decryption_failure_count_ = 0; + has_valid_key_ = true; + } + + void SetKeyFromMaterial(std::vector password, int key_index) { + webrtc::MutexLock lock(&mutex_); + if (key_index >= 0) { + current_key_index_ = key_index % crypto_key_ring_.size(); + } + crypto_key_ring_[current_key_index_] = + DeriveKeys(password, key_provider_->options().ratchet_salt, 128); + } + + bool DecryptionFailure() { + webrtc::MutexLock lock(&mutex_); + if (key_provider_->options().failure_tolerance < 0) { + return false; + } + decryption_failure_count_ += 1; + + if (decryption_failure_count_ > + key_provider_->options().failure_tolerance) { + has_valid_key_ = false; + return true; + } + return false; + } + + private: + bool has_valid_key_ = false; + int decryption_failure_count_ = 0; + mutable webrtc::Mutex mutex_; + int current_key_index_ = 0; + KeyProvider* key_provider_; + std::vector> crypto_key_ring_; +}; + +class DefaultKeyProviderImpl : public KeyProvider { + public: + DefaultKeyProviderImpl(KeyProviderOptions options) : options_(options) {} + ~DefaultKeyProviderImpl() override = default; + + /// Set the shared key. + bool SetSharedKey(int key_index, std::vector key) override { + webrtc::MutexLock lock(&mutex_); + if (options_.shared_key) { + if (keys_.find("shared") == keys_.end()) { + keys_["shared"] = rtc::make_ref_counted(this); + } + + auto key_handler = keys_["shared"]; + key_handler->SetKey(key, key_index); + + for (auto& key_pair : keys_) { + if (key_pair.first != "shared") { + key_pair.second->SetKey(key, key_index); + } + } + return true; + } + return false; + } + + const std::vector RatchetSharedKey(int key_index) override { + webrtc::MutexLock lock(&mutex_); + auto it = keys_.find("shared"); + if (it == keys_.end()) { + return std::vector(); + } + auto new_key = it->second->RatchetKey(key_index); + if (options_.shared_key) { + for (auto& key_pair : keys_) { + if (key_pair.first != "shared") { + key_pair.second->SetKey(new_key, key_index); + } + } + } + return new_key; + } + + const std::vector ExportSharedKey(int key_index) const override { + webrtc::MutexLock lock(&mutex_); + auto it = keys_.find("shared"); + if (it == keys_.end()) { + return std::vector(); + } + auto key_set = it->second->GetKeySet(key_index); + if (key_set) { + return key_set->material; + } + return std::vector(); + } + + const rtc::scoped_refptr GetSharedKey( + const std::string participant_id) override { + webrtc::MutexLock lock(&mutex_); + if (options_.shared_key && keys_.find("shared") != keys_.end()) { + auto shared_key_handler = keys_["shared"]; + if (keys_.find(participant_id) != keys_.end()) { + return keys_[participant_id]; + } else { + auto key_handler_clone = shared_key_handler->Clone(); + keys_[participant_id] = key_handler_clone; + return key_handler_clone; + } + } + return nullptr; + } + + /// Set the key at the given index. + bool SetKey(const std::string participant_id, + int index, + std::vector key) override { + webrtc::MutexLock lock(&mutex_); + + if (keys_.find(participant_id) == keys_.end()) { + keys_[participant_id] = + rtc::make_ref_counted(this); + } + + auto key_handler = keys_[participant_id]; + key_handler->SetKey(key, index); + return true; + } + + const rtc::scoped_refptr GetKey( + const std::string participant_id) const override { + webrtc::MutexLock lock(&mutex_); + + if (keys_.find(participant_id) == keys_.end()) { + return nullptr; + } + + return keys_.find(participant_id)->second; + } + + const std::vector RatchetKey(const std::string participant_id, + int key_index) override { + auto key_handler = GetKey(participant_id); + if (key_handler) { + return key_handler->RatchetKey(key_index); + } + return std::vector(); + } + + const std::vector ExportKey(const std::string participant_id, + int key_index) const override { + auto key_handler = GetKey(participant_id); + if (key_handler) { + auto key_set = key_handler->GetKeySet(key_index); + if (key_set) { + return key_set->material; + } + } + return std::vector(); + } + + void SetSifTrailer(const std::vector trailer) override { + webrtc::MutexLock lock(&mutex_); + options_.uncrypted_magic_bytes = trailer; + } + + KeyProviderOptions& options() override { return options_; } + + private: + mutable webrtc::Mutex mutex_; + KeyProviderOptions options_; + std::unordered_map> + keys_; +}; + +enum FrameCryptionState { + kNew = 0, + kOk, + kEncryptionFailed, + kDecryptionFailed, + kMissingKey, + kKeyRatcheted, + kInternalError, +}; + +class FrameCryptorTransformerObserver : public rtc::RefCountInterface { + public: + virtual void OnFrameCryptionStateChanged(const std::string participant_id, + FrameCryptionState error) = 0; + + protected: + virtual ~FrameCryptorTransformerObserver() {} +}; + +class RTC_EXPORT FrameCryptorTransformer + : public rtc::RefCountedObject { + public: + enum class MediaType { + kAudioFrame = 0, + kVideoFrame, + }; + + enum class Algorithm { + kAesGcm = 0, + kAesCbc, + }; + + explicit FrameCryptorTransformer( + rtc::Thread* signaling_thread, + const std::string participant_id, + MediaType type, + Algorithm algorithm, + rtc::scoped_refptr key_provider); + ~FrameCryptorTransformer(); + virtual void RegisterFrameCryptorTransformerObserver( + rtc::scoped_refptr observer) { + webrtc::MutexLock lock(&mutex_); + observer_ = observer; + } + + virtual void UnRegisterFrameCryptorTransformerObserver() { + webrtc::MutexLock lock(&mutex_); + observer_ = nullptr; + } + + virtual void SetKeyIndex(int index) { + webrtc::MutexLock lock(&mutex_); + key_index_ = index; + } + + virtual int key_index() const { return key_index_; } + + virtual void SetEnabled(bool enabled) { + webrtc::MutexLock lock(&mutex_); + enabled_cryption_ = enabled; + } + virtual bool enabled() const { + webrtc::MutexLock lock(&mutex_); + return enabled_cryption_; + } + virtual const std::string participant_id() const { return participant_id_; } + + protected: + virtual void RegisterTransformedFrameCallback( + rtc::scoped_refptr callback) override { + webrtc::MutexLock lock(&sink_mutex_); + sink_callback_ = callback; + } + virtual void UnregisterTransformedFrameCallback() override { + webrtc::MutexLock lock(&sink_mutex_); + sink_callback_ = nullptr; + } + virtual void RegisterTransformedFrameSinkCallback( + rtc::scoped_refptr callback, + uint32_t ssrc) override { + webrtc::MutexLock lock(&sink_mutex_); + sink_callbacks_[ssrc] = callback; + } + virtual void UnregisterTransformedFrameSinkCallback(uint32_t ssrc) override { + webrtc::MutexLock lock(&sink_mutex_); + auto it = sink_callbacks_.find(ssrc); + if (it != sink_callbacks_.end()) { + sink_callbacks_.erase(it); + } + } + + virtual void Transform( + std::unique_ptr frame) override; + + private: + void encryptFrame(std::unique_ptr frame); + void decryptFrame(std::unique_ptr frame); + void onFrameCryptionStateChanged(FrameCryptionState error); + rtc::Buffer makeIv(uint32_t ssrc, uint32_t timestamp); + uint8_t getIvSize(); + + private: + TaskQueueBase* const signaling_thread_; + std::unique_ptr thread_; + std::string participant_id_; + mutable webrtc::Mutex mutex_; + mutable webrtc::Mutex sink_mutex_; + bool enabled_cryption_ RTC_GUARDED_BY(mutex_) = false; + MediaType type_; + Algorithm algorithm_; + rtc::scoped_refptr sink_callback_; + std::map> + sink_callbacks_; + int key_index_ = 0; + std::map send_counts_; + rtc::scoped_refptr key_provider_; + rtc::scoped_refptr observer_; + FrameCryptionState last_enc_error_ = FrameCryptionState::kNew; + FrameCryptionState last_dec_error_ = FrameCryptionState::kNew; +}; + +} // namespace webrtc + +#endif // WEBRTC_FRAME_CRYPTOR_TRANSFORMER_H_ diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 102ac33334..3640cbccc1 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -64,6 +64,8 @@ class TransformableVideoFrameInterface : public TransformableFrameInterface { virtual VideoFrameMetadata Metadata() const = 0; virtual void SetMetadata(const VideoFrameMetadata&) = 0; + + virtual const RTPVideoHeader& header () const = 0; }; // Extends the TransformableFrameInterface to expose audio-specific information. diff --git a/api/peer_connection_interface.h b/api/peer_connection_interface.h index ca6baacd0d..60e071fef4 100644 --- a/api/peer_connection_interface.h +++ b/api/peer_connection_interface.h @@ -680,6 +680,15 @@ class RTC_EXPORT PeerConnectionInterface : public webrtc::RefCountInterface { // The burst interval of the pacer, see TaskQueuePacedSender constructor. absl::optional pacer_burst_interval; + // When this flag is set, ports not bound to any specific network interface + // will be used, in addition to normal ports bound to the enumerated + // interfaces. Without this flag, these "any address" ports would only be + // used when network enumeration fails or is disabled. But under certain + // conditions, these ports may succeed where others fail, so they may allow + // the application to work in a wider variety of environments, at the expense + // of having to allocate additional candidates. + bool enable_any_address_ports = false; + // // Don't forget to update operator== if adding something. // diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 5ec689c096..60cd07c512 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -81,6 +81,19 @@ rtc_library("video_frame") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +rtc_library("yuv_helper") { + visibility = [ "*" ] + sources = [ + "yuv_helper.cc", + "yuv_helper.h", + ] + + deps = [ + "../../rtc_base/system:rtc_export", + "//third_party/libyuv", + ] +} + if (is_android) { java_cpp_enum("video_frame_enums") { sources = [ "video_frame_buffer.h" ] diff --git a/api/video/yuv_helper.cc b/api/video/yuv_helper.cc new file mode 100644 index 0000000000..eab9126183 --- /dev/null +++ b/api/video/yuv_helper.cc @@ -0,0 +1,416 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "yuv_helper.h" + +#include "libyuv/convert.h" +#include "libyuv/convert_argb.h" +#include "libyuv/convert_from_argb.h" +#include "libyuv/row.h" +#include "third_party/libyuv/include/libyuv.h" +#include "video_rotation.h" + +namespace webrtc { + +int I420Rotate(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height, + VideoRotation mode) { + return libyuv::I420Rotate(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, width, height, + static_cast(mode)); +} + +int I420ToNV12(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height) { + return libyuv::I420ToNV12(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_y, dst_stride_y, dst_uv, + dst_stride_uv, width, height); +} + +int I420ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + return libyuv::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_argb, dst_stride_argb, width, + height); +} + +int I420ToBGRA(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_bgra, + int dst_stride_bgra, + int width, + int height) { + return libyuv::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_bgra, dst_stride_bgra, width, + height); +} + +int I420ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + return libyuv::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_abgr, dst_stride_abgr, width, + height); +} + +int I420ToRGBA(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_rgba, + int dst_stride_rgba, + int width, + int height) { + return libyuv::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_rgba, dst_stride_rgba, width, + height); +} + +int I420ToRGB24(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_rgb24, + int dst_stride_rgb24, + int width, + int height) { + return libyuv::I420ToRGB24(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_rgb24, dst_stride_rgb24, width, + height); +} + +int I420Scale(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + int src_width, + int src_height, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int dst_width, + int dst_height, + libyuv::FilterMode filtering) { + return libyuv::I420Scale(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, src_width, src_height, dst_y, + dst_stride_y, dst_u, dst_stride_u, dst_v, + dst_stride_v, dst_width, dst_height, filtering); +} + +int ARGBToI420(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return libyuv::ARGBToI420(src_argb, src_stride_argb, dst_y, dst_stride_y, + dst_u, dst_stride_u, dst_v, dst_stride_v, width, + height); +} + +int ABGRToI420(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return libyuv::ABGRToI420(src_abgr, src_stride_abgr, dst_y, dst_stride_y, + dst_u, dst_stride_u, dst_v, dst_stride_v, width, + height); +} + +int ARGBToRGB24(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_rgb24, + int dst_stride_rgb24, + int width, + int height) { + return libyuv::ARGBToRGB24(src_argb, src_stride_argb, dst_rgb24, + dst_stride_rgb24, width, height); +} + +int NV12ToI420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return libyuv::NV12ToI420(src_y, src_stride_y, src_uv, src_stride_uv, dst_y, + dst_stride_y, dst_u, dst_stride_u, dst_v, + dst_stride_v, width, height); +} + +int I444ToI420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return libyuv::I444ToI420(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, width, height); +} + +int I422ToI420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return libyuv::I422ToI420(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, width, height); +} + +int I010ToI420(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + return libyuv::I010ToI420(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_y, dst_stride_y, dst_u, + dst_stride_u, dst_v, dst_stride_v, width, height); +} + +int NV12ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + return libyuv::NV12ToARGB(src_y, src_stride_y, src_uv, src_stride_uv, + dst_argb, dst_stride_argb, width, height); +} + +int NV12ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + return libyuv::NV12ToABGR(src_y, src_stride_y, src_uv, src_stride_uv, + dst_abgr, dst_stride_abgr, width, height); +} + +int I444ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + return libyuv::I444ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_abgr, dst_stride_abgr, width, + height); +} + +int I444ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + return libyuv::I444ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_abgr, dst_stride_abgr, width, + height); +} + +int I422ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + return libyuv::I422ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_argb, dst_stride_argb, width, + height); +} + +int I422ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + return libyuv::I422ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_abgr, dst_stride_abgr, width, + height); +} + +int I010ToARGB(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + return libyuv::I010ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_argb, dst_stride_argb, width, + height); +} + +int I010ToABGR(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + return libyuv::I010ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_abgr, dst_stride_abgr, width, + height); +} + +int ABGRToNV12(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height) { + return libyuv::ABGRToNV12(src_abgr, src_stride_abgr, dst_y, dst_stride_y, + dst_uv, dst_stride_uv, width, height); +} + +int ARGBToNV12(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height) { + return libyuv::ARGBToNV12(src_argb, src_stride_argb, dst_y, dst_stride_y, + dst_uv, dst_stride_uv, width, height); +} + +} // namespace webrtc diff --git a/api/video/yuv_helper.h b/api/video/yuv_helper.h new file mode 100644 index 0000000000..5e86fb378b --- /dev/null +++ b/api/video/yuv_helper.h @@ -0,0 +1,316 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "libyuv/convert.h" +#include "rtc_base/system/rtc_export.h" +#include "stdint.h" +#include "third_party/libyuv/include/libyuv.h" +#include "video_rotation.h" + +namespace webrtc { + +RTC_EXPORT int I420Rotate(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height, + VideoRotation mode); + +RTC_EXPORT int I420ToNV12(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height); + +RTC_EXPORT int I420ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height); + +RTC_EXPORT int I420ToBGRA(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_bgra, + int dst_stride_bgra, + int width, + int height); + +RTC_EXPORT int I420ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height); + +RTC_EXPORT int I420ToRGBA(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_rgba, + int dst_stride_rgba, + int width, + int height); + +RTC_EXPORT int I420ToRGB24(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_rgb24, + int dst_stride_rgb24, + int width, + int height); + +RTC_EXPORT int I420Scale(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + int src_width, + int src_height, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int dst_width, + int dst_height, + libyuv::FilterMode filtering); + +RTC_EXPORT int ARGBToI420(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + +RTC_EXPORT int ABGRToI420(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + +RTC_EXPORT int ARGBToRGB24(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_rgb24, + int dst_stride_rgb24, + int width, + int height); + +RTC_EXPORT int NV12ToI420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + +RTC_EXPORT int I444ToI420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + +RTC_EXPORT int I422ToI420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + +RTC_EXPORT int I010ToI420(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height); + +RTC_EXPORT int NV12ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height); + +RTC_EXPORT int NV12ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height); + +RTC_EXPORT int I444ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height); + +RTC_EXPORT int I444ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height); + +RTC_EXPORT int I422ToARGB(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height); + +RTC_EXPORT int I422ToABGR(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height); + +RTC_EXPORT int I010ToARGB(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height); + +RTC_EXPORT int I010ToABGR(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height); + +RTC_EXPORT int ABGRToNV12(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height); + +RTC_EXPORT int ARGBToNV12(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height); + +} // namespace webrtc diff --git a/api/video_codecs/video_encoder_factory.h b/api/video_codecs/video_encoder_factory.h index d6e7ff93c5..ddc4eb7cb8 100644 --- a/api/video_codecs/video_encoder_factory.h +++ b/api/video_codecs/video_encoder_factory.h @@ -85,13 +85,8 @@ class VideoEncoderFactory { virtual CodecSupport QueryCodecSupport( const SdpVideoFormat& format, absl::optional scalability_mode) const { - // Default implementation, query for supported formats and check if the - // specified format is supported. Returns false if scalability_mode is - // specified. CodecSupport codec_support; - if (!scalability_mode) { - codec_support.is_supported = format.IsCodecInList(GetSupportedFormats()); - } + codec_support.is_supported = format.IsCodecInList(GetSupportedFormats()); return codec_support; } diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index 17cf859ed8..d94ac6a70c 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -362,7 +362,6 @@ void ChannelReceive::OnReceivedPayloadData( void ChannelReceive::InitFrameTransformerDelegate( rtc::scoped_refptr frame_transformer) { RTC_DCHECK(frame_transformer); - RTC_DCHECK(!frame_transformer_delegate_); RTC_DCHECK(worker_thread_->IsCurrent()); // Pass a callback to ChannelReceive::OnReceivedPayloadData, to be called by @@ -922,6 +921,9 @@ void ChannelReceive::SetDepacketizerToDecoderFrameTransformer( RTC_DCHECK_NOTREACHED() << "Not setting the transformer?"; return; } + if(frame_transformer_delegate_) { + frame_transformer_delegate_->Reset(); + } if (frame_transformer_delegate_) { // Depending on when the channel is created, the transformer might be set // twice. Don't replace the delegate if it was already initialized. diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index a78cf6a564..f3eb90c1ec 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -3518,15 +3518,22 @@ void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetReceiverParameters( } } -void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StartStream(){ - if (stream_) { - stream_->Start(); +void WebRtcVideoReceiveChannel::StartReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; } + stream->StartReceiveStream(); } -void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StopStream(){ - if (stream_) { - stream_->Stop(); + +void WebRtcVideoReceiveChannel::StopReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; } + stream->StopReceiveStream(); } void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index f9636c708a..d3bcedb495 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -631,7 +631,6 @@ class WebRtcVideoReceiveChannel : public MediaChannelUtil, bool nack_enabled, webrtc::RtcpMode rtcp_mode, absl::optional rtx_time) override; - void StartReceive(uint32_t ssrc) override; void StopReceive(uint32_t ssrc) override; private: diff --git a/modules/audio_device/include/audio_device.h b/modules/audio_device/include/audio_device.h index a573f5a5f9..58019cc24f 100644 --- a/modules/audio_device/include/audio_device.h +++ b/modules/audio_device/include/audio_device.h @@ -12,10 +12,10 @@ #define MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_H_ #include "absl/types/optional.h" -#include "api/ref_count.h" #include "api/scoped_refptr.h" #include "api/task_queue/task_queue_factory.h" #include "modules/audio_device/include/audio_device_defines.h" +#include "rtc_base/ref_count.h" namespace webrtc { diff --git a/modules/desktop_capture/mac/screen_capturer_mac.mm b/modules/desktop_capture/mac/screen_capturer_mac.mm index 60089fd0f2..aa05ceb9aa 100644 --- a/modules/desktop_capture/mac/screen_capturer_mac.mm +++ b/modules/desktop_capture/mac/screen_capturer_mac.mm @@ -216,16 +216,7 @@ DesktopRect GetExcludedWindowPixelBounds(CGWindowID window, float dip_to_pixel_s ScreenConfigurationChanged(); } - // When screen is zoomed in/out, OSX only updates the part of Rects currently - // displayed on screen, with relative location to current top-left on screen. - // This will cause problems when we copy the dirty regions to the captured - // image. So we invalidate the whole screen to copy all the screen contents. - // With CGI method, the zooming will be ignored and the whole screen contents - // will be captured as before. - // With IOSurface method, the zoomed screen contents will be captured. - if (UAZoomEnabled()) { - helper_.InvalidateScreen(screen_pixel_bounds_.size()); - } + helper_.InvalidateScreen(screen_pixel_bounds_.size()); DesktopRegion region; helper_.TakeInvalidRegion(®ion); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 2bb71941f9..106adc0d39 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -92,6 +92,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { csrcs_ = metadata.GetCsrcs(); } + const RTPVideoHeader& header() const override { return header_; } + const RTPVideoHeader& GetHeader() const { return header_; } uint8_t GetPayloadType() const override { return payload_type_; } absl::optional GetCodecType() const { return codec_type_; } diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc index fbd10c4c7b..4f24f4842f 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -72,6 +72,10 @@ class TransformableVideoReceiverFrame frame_->SetHeaderFromMetadata(new_metadata); } + const RTPVideoHeader& header () const override { + return frame_->GetRtpVideoHeader(); + } + std::unique_ptr ExtractFrame() && { return std::move(frame_); } diff --git a/pc/audio_rtp_receiver.cc b/pc/audio_rtp_receiver.cc index 6e7ca6d0b5..ae879ddb1b 100644 --- a/pc/audio_rtp_receiver.cc +++ b/pc/audio_rtp_receiver.cc @@ -298,7 +298,7 @@ void AudioRtpReceiver::Reconfigure(bool track_enabled) { media_channel_->SetFrameDecryptor(*signaled_ssrc_, frame_decryptor_); } - if (frame_transformer_) { + if (frame_transformer_ && track_enabled) { media_channel_->SetDepacketizerToDecoderFrameTransformer( signaled_ssrc_.value_or(0), frame_transformer_); } diff --git a/pc/peer_connection.cc b/pc/peer_connection.cc index 444a0bf618..950b71254e 100644 --- a/pc/peer_connection.cc +++ b/pc/peer_connection.cc @@ -445,6 +445,7 @@ bool PeerConnectionInterface::RTCConfiguration::operator==( std::vector vpn_list; PortAllocatorConfig port_allocator_config; absl::optional pacer_burst_interval; + bool enable_any_address_ports; }; static_assert(sizeof(stuff_being_tested_for_equality) == sizeof(*this), "Did you add something to RTCConfiguration and forget to " @@ -506,7 +507,8 @@ bool PeerConnectionInterface::RTCConfiguration::operator==( port_allocator_config.min_port == o.port_allocator_config.min_port && port_allocator_config.max_port == o.port_allocator_config.max_port && port_allocator_config.flags == o.port_allocator_config.flags && - pacer_burst_interval == o.pacer_burst_interval; + pacer_burst_interval == o.pacer_burst_interval && + enable_any_address_ports == o.enable_any_address_ports; } bool PeerConnectionInterface::RTCConfiguration::operator!=( @@ -2148,6 +2150,11 @@ PeerConnection::InitializePortAllocator_n( RTC_LOG(LS_INFO) << "Disable candidates on link-local network interfaces."; } + if (configuration.enable_any_address_ports) { + port_allocator_flags |= cricket::PORTALLOCATOR_ENABLE_ANY_ADDRESS_PORTS; + RTC_LOG(LS_INFO) << "Enable gathering on any address ports."; + } + port_allocator_->set_flags(port_allocator_flags); // No step delay is used while allocating ports. port_allocator_->set_step_delay(cricket::kMinimumStepDelay); diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index d3fffc3bea..7abe979702 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -119,6 +119,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrame.mm", "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", + "objc/base/RTCAudioRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", ] @@ -143,12 +144,15 @@ if (is_ios || is_mac) { "objc/helpers/RTCDispatcher+Private.h", "objc/helpers/RTCDispatcher.h", "objc/helpers/RTCDispatcher.m", + "objc/helpers/RTCYUVHelper.h", + "objc/helpers/RTCYUVHelper.mm", "objc/helpers/scoped_cftyperef.h", ] deps = [ ":base_objc", "../rtc_base:checks", + "//third_party/libyuv", ] absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] @@ -407,7 +411,12 @@ if (is_ios || is_mac) { "objc/components/network/RTCNetworkMonitor.mm", ] - configs += [ ":used_from_extension" ] + configs += [ + "..:common_objc", + ":used_from_extension", + ] + + public_configs = [ ":common_config_objc" ] frameworks = [ "Network.framework" ] @@ -616,17 +625,13 @@ if (is_ios || is_mac) { "Metal.framework", "MetalKit.framework", ] - if (is_ios) { + if (is_ios || is_mac) { sources += [ "objc/components/renderer/metal/RTCMTLVideoView.h", "objc/components/renderer/metal/RTCMTLVideoView.m", ] } if (is_mac) { - sources += [ - "objc/components/renderer/metal/RTCMTLNSVideoView.h", - "objc/components/renderer/metal/RTCMTLNSVideoView.m", - ] frameworks += [ "AppKit.framework" ] } deps = [ @@ -682,6 +687,43 @@ if (is_ios || is_mac) { "../rtc_base/system:gcd_helpers", ] } + + rtc_library("desktopcapture_objc") { + visibility = [ "*" ] + sources = [ + "objc/components/capturer/RTCDesktopCapturer+Private.h", + "objc/components/capturer/RTCDesktopCapturer.h", + "objc/components/capturer/RTCDesktopCapturer.mm", + "objc/components/capturer/RTCDesktopSource+Private.h", + "objc/components/capturer/RTCDesktopSource.h", + "objc/components/capturer/RTCDesktopSource.mm", + "objc/components/capturer/RTCDesktopMediaList+Private.h", + "objc/components/capturer/RTCDesktopMediaList.h", + "objc/components/capturer/RTCDesktopMediaList.mm", + "objc/native/src/objc_desktop_capture.h", + "objc/native/src/objc_desktop_capture.mm", + "objc/native/src/objc_desktop_media_list.h", + "objc/native/src/objc_desktop_media_list.mm", + ] + frameworks = [ + "AppKit.framework", + ] + + configs += [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":helpers_objc", + ":videoframebuffer_objc", + "../rtc_base/system:gcd_helpers", + "../modules/desktop_capture", + ] + if(is_mac) { + deps += [ "//third_party:jpeg", ] + } + } rtc_library("videocodec_objc") { visibility = [ "*" ] @@ -838,6 +880,22 @@ if (is_ios || is_mac) { ] } + rtc_library("simulcast") { + sources = [ + "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h", + "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm", + "objc/api/video_codec/RTCVideoEncoderSimulcast.h", + "objc/api/video_codec/RTCVideoEncoderSimulcast.mm", + ] + + deps = [ + ":base_objc", + ":wrapped_native_codec_objc", + "../media:rtc_media_base", + "../media:rtc_simulcast_encoder_adapter", + ] + } + rtc_library("mediaconstraints_objc") { configs += [ "..:no_global_constructors" ] sources = [ @@ -969,6 +1027,12 @@ if (is_ios || is_mac) { "objc/api/peerconnection/RTCDtmfSender.mm", "objc/api/peerconnection/RTCFieldTrials.h", "objc/api/peerconnection/RTCFieldTrials.mm", + "objc/api/peerconnection/RTCFrameCryptor+Private.h", + "objc/api/peerconnection/RTCFrameCryptor.h", + "objc/api/peerconnection/RTCFrameCryptor.mm", + "objc/api/peerconnection/RTCFrameCryptorKeyProvider+Private.h", + "objc/api/peerconnection/RTCFrameCryptorKeyProvider.h", + "objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm", "objc/api/peerconnection/RTCIceCandidate+Private.h", "objc/api/peerconnection/RTCIceCandidate.h", "objc/api/peerconnection/RTCIceCandidate.mm", @@ -1093,6 +1157,7 @@ if (is_ios || is_mac) { ":videorendereradapter_objc", ":videosource_objc", ":videotoolbox_objc", + "../api/crypto:frame_crypto_transformer", "../api:dtmf_sender_interface", "../api:enable_media", "../api:libjingle_peerconnection_api", @@ -1286,6 +1351,13 @@ if (is_ios || is_mac) { } } + bundle_data("darwin_privacy_info") { + sources = [ + "objc/PrivacyInfo.xcprivacy", + ] + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + if (is_ios) { apple_framework_bundle_with_umbrella_header("framework_objc") { info_plist = "objc/Info.plist" @@ -1333,6 +1405,7 @@ if (is_ios || is_mac) { "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", "objc/helpers/RTCCameraPreviewView.h", "objc/helpers/RTCDispatcher.h", + "objc/helpers/RTCYUVHelper.h", "objc/helpers/UIDevice+RTCDevice.h", "objc/api/peerconnection/RTCAudioDeviceModule.h", "objc/api/peerconnection/RTCIODevice.h", @@ -1341,6 +1414,8 @@ if (is_ios || is_mac) { "objc/api/peerconnection/RTCConfiguration.h", "objc/api/peerconnection/RTCDataChannel.h", "objc/api/peerconnection/RTCDataChannelConfiguration.h", + "objc/api/peerconnection/RTCFrameCryptor.h", + "objc/api/peerconnection/RTCFrameCryptorKeyProvider.h", "objc/api/peerconnection/RTCFieldTrials.h", "objc/api/peerconnection/RTCIceCandidate.h", "objc/api/peerconnection/RTCIceCandidateErrorEvent.h", @@ -1421,6 +1496,7 @@ if (is_ios || is_mac) { ":videocapture_objc", ":videocodec_objc", ":videotoolbox_objc", + ":darwin_privacy_info", ] if (!build_with_chromium) { deps += [ @@ -1468,6 +1544,8 @@ if (is_ios || is_mac) { "objc/api/peerconnection/RTCDataChannel.h", "objc/api/peerconnection/RTCDataChannelConfiguration.h", "objc/api/peerconnection/RTCDtmfSender.h", + "objc/api/peerconnection/RTCFrameCryptor.h", + "objc/api/peerconnection/RTCFrameCryptorKeyProvider.h", "objc/api/peerconnection/RTCFieldTrials.h", "objc/api/peerconnection/RTCIceCandidate.h", "objc/api/peerconnection/RTCIceCandidateErrorEvent.h", @@ -1530,6 +1608,10 @@ if (is_ios || is_mac) { "objc/base/RTCYUVPlanarBuffer.h", "objc/components/capturer/RTCCameraVideoCapturer.h", "objc/components/capturer/RTCFileVideoCapturer.h", + "objc/components/capturer/RTCDesktopCapturer.h", + "objc/components/capturer/RTCDesktopSource.h", + "objc/components/capturer/RTCDesktopMediaList.h", + "objc/components/renderer/metal/RTCMTLVideoView.h", "objc/components/renderer/metal/RTCMTLNSVideoView.h", "objc/components/renderer/opengl/RTCVideoViewShading.h", "objc/components/video_codec/RTCCodecSpecificInfoH264.h", @@ -1565,8 +1647,10 @@ if (is_ios || is_mac) { ":native_video", ":peerconnectionfactory_base_objc", ":videocapture_objc", + ":desktopcapture_objc", ":videocodec_objc", ":videotoolbox_objc", + ":darwin_privacy_info", ] if (!build_with_chromium) { deps += [ diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 96ffa7dc0a..715dc49e6f 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -264,12 +264,18 @@ if (is_android) { "api/org/webrtc/AudioProcessingFactory.java", "api/org/webrtc/AudioSource.java", "api/org/webrtc/AudioTrack.java", + "api/org/webrtc/AudioTrackSink.java", "api/org/webrtc/CallSessionFileRotatingLogSink.java", "api/org/webrtc/CandidatePairChangeEvent.java", "api/org/webrtc/CryptoOptions.java", "api/org/webrtc/DataChannel.java", "api/org/webrtc/DtmfSender.java", + "api/org/webrtc/ExternalAudioProcessingFactory.java", "api/org/webrtc/FecControllerFactoryFactoryInterface.java", + "api/org/webrtc/FrameCryptor.java", + "api/org/webrtc/FrameCryptorAlgorithm.java", + "api/org/webrtc/FrameCryptorFactory.java", + "api/org/webrtc/FrameCryptorKeyProvider.java", "api/org/webrtc/FrameDecryptor.java", "api/org/webrtc/FrameEncryptor.java", "api/org/webrtc/IceCandidate.java", @@ -730,6 +736,8 @@ if (current_os == "linux" || is_android) { "src/jni/pc/add_ice_candidate_observer.cc", "src/jni/pc/add_ice_candidate_observer.h", "src/jni/pc/android_network_monitor.h", + "src/jni/pc/audio_sink.cc", + "src/jni/pc/audio_sink.h", "src/jni/pc/audio_track.cc", "src/jni/pc/call_session_file_rotating_log_sink.cc", "src/jni/pc/crypto_options.cc", @@ -737,6 +745,15 @@ if (current_os == "linux" || is_android) { "src/jni/pc/data_channel.cc", "src/jni/pc/data_channel.h", "src/jni/pc/dtmf_sender.cc", + "src/jni/pc/external_audio_processing_factory.cc", + "src/jni/pc/external_audio_processing_factory.h", + "src/jni/pc/external_audio_processing_interface.h", + "src/jni/pc/external_audio_processor.cc", + "src/jni/pc/external_audio_processor.h", + "src/jni/pc/frame_cryptor.cc", + "src/jni/pc/frame_cryptor.h", + "src/jni/pc/frame_cryptor_key_provider.cc", + "src/jni/pc/frame_cryptor_key_provider.h", "src/jni/pc/ice_candidate.cc", "src/jni/pc/ice_candidate.h", "src/jni/pc/media_constraints.cc", @@ -795,6 +812,7 @@ if (current_os == "linux" || is_android) { "../../api:rtp_parameters", "../../api:rtp_sender_interface", "../../api:turn_customizer", + "../../api/crypto:frame_crypto_transformer", "../../api/crypto:options", "../../api/rtc_event_log:rtc_event_log_factory", "../../api/task_queue:default_task_queue_factory", @@ -929,6 +947,21 @@ if (current_os == "linux" || is_android) { ] } + rtc_library("simulcast_jni") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "src/jni/simulcast_video_encoder.cc", + "src/jni/simulcast_video_encoder.h" + ] + deps = [ + ":base_jni", + ":video_jni", + ":native_api_codecs", + "../../media:rtc_simulcast_encoder_adapter" + ] + } + rtc_library("swcodecs_jni") { visibility = [ "*" ] allow_poison = [ "software_video_codecs" ] @@ -943,6 +976,7 @@ if (current_os == "linux" || is_android) { ":libvpx_vp9_jni", ":simulcast_jni", ":native_api_jni", + ":simulcast_jni", ":video_jni", "../../api/environment", "../../api/video_codecs:builtin_video_decoder_factory", @@ -1427,11 +1461,16 @@ if (current_os == "linux" || is_android) { sources = [ "api/org/webrtc/AddIceObserver.java", "api/org/webrtc/AudioTrack.java", + "api/org/webrtc/AudioTrackSink.java", "api/org/webrtc/CallSessionFileRotatingLogSink.java", "api/org/webrtc/CandidatePairChangeEvent.java", "api/org/webrtc/CryptoOptions.java", "api/org/webrtc/DataChannel.java", "api/org/webrtc/DtmfSender.java", + "api/org/webrtc/ExternalAudioProcessingFactory.java", + "api/org/webrtc/FrameCryptor.java", + "api/org/webrtc/FrameCryptorFactory.java", + "api/org/webrtc/FrameCryptorKeyProvider.java", "api/org/webrtc/IceCandidate.java", "api/org/webrtc/IceCandidateErrorEvent.java", "api/org/webrtc/MediaConstraints.java", diff --git a/sdk/android/api/org/webrtc/AudioTrack.java b/sdk/android/api/org/webrtc/AudioTrack.java index ca745db634..b30e46cebc 100644 --- a/sdk/android/api/org/webrtc/AudioTrack.java +++ b/sdk/android/api/org/webrtc/AudioTrack.java @@ -10,8 +10,12 @@ package org.webrtc; +import java.util.IdentityHashMap; + /** Java wrapper for a C++ AudioTrackInterface */ public class AudioTrack extends MediaStreamTrack { + private final IdentityHashMap sinks = new IdentityHashMap(); + public AudioTrack(long nativeTrack) { super(nativeTrack); } @@ -23,10 +27,54 @@ public void setVolume(double volume) { nativeSetVolume(getNativeAudioTrack(), volume); } + /** + * Adds an AudioTrackSink to the track. This callback is only + * called for remote audio tracks. + * + * Repeated addSink calls will not add the sink multiple times. + */ + public void addSink(AudioTrackSink sink) { + if (sink == null) { + throw new IllegalArgumentException("The AudioTrackSink is not allowed to be null"); + } + if (!sinks.containsKey(sink)) { + final long nativeSink = nativeWrapSink(sink); + sinks.put(sink, nativeSink); + nativeAddSink(getNativeMediaStreamTrack(), nativeSink); + } + } + + /** + * Removes an AudioTrackSink from the track. + * + * If the AudioTrackSink was not attached to the track, this is a no-op. + */ + public void removeSink(AudioTrackSink sink) { + final Long nativeSink = sinks.remove(sink); + if (nativeSink != null) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + } + + @Override + public void dispose() { + for (long nativeSink : sinks.values()) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + sinks.clear(); + super.dispose(); + } + /** Returns a pointer to webrtc::AudioTrackInterface. */ long getNativeAudioTrack() { return getNativeMediaStreamTrack(); } private static native void nativeSetVolume(long track, double volume); + private static native void nativeAddSink(long track, long nativeSink); + private static native void nativeRemoveSink(long track, long nativeSink); + private static native long nativeWrapSink(AudioTrackSink sink); + private static native void nativeFreeSink(long sink); } diff --git a/sdk/android/api/org/webrtc/AudioTrackSink.java b/sdk/android/api/org/webrtc/AudioTrackSink.java new file mode 100644 index 0000000000..eca390f82c --- /dev/null +++ b/sdk/android/api/org/webrtc/AudioTrackSink.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** + * Java version of rtc::AudioTrackSinkInterface. + */ +public interface AudioTrackSink { + /** + * Implementations should copy the audio data into a local copy if they wish + * to use the data after this function returns. + */ + @CalledByNative + void onData(ByteBuffer audioData, int bitsPerSample, int sampleRate, + int numberOfChannels, int numberOfFrames, + long absoluteCaptureTimestampMs); +} diff --git a/sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java b/sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java new file mode 100644 index 0000000000..7425d2af57 --- /dev/null +++ b/sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java @@ -0,0 +1,144 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +import androidx.annotation.Nullable; +import org.webrtc.AudioProcessingFactory; + + +public class ExternalAudioProcessingFactory implements AudioProcessingFactory { + + /** + * Interface for external audio processing. + */ + public static interface AudioProcessing { + /** + * Called when the processor should be initialized with a new sample rate and + * number of channels. + */ + @CalledByNative("AudioProcessing") + void initialize(int sampleRateHz, int numChannels); + /** Called when the processor should be reset with a new sample rate. */ + @CalledByNative("AudioProcessing") + void reset(int newRate); + /** + * Processes the given capture or render signal. NOTE: `buffer.data` will be + * freed once this function returns so callers who want to use the data + * asynchronously must make sure to copy it first. + */ + @CalledByNative("AudioProcessing") + void process(int numBands, int numFrames, ByteBuffer buffer); + } + + private long apmPtr; + private long capturePostProcessingPtr; + private long renderPreProcessingPtr; + + public ExternalAudioProcessingFactory() { + apmPtr = nativeGetDefaultApm(); + capturePostProcessingPtr = 0; + renderPreProcessingPtr = 0; + } + + @Override + public long createNative() { + if(apmPtr == 0) { + apmPtr = nativeGetDefaultApm(); + } + return apmPtr; + } + + /** + * Sets the capture post processing module. + * This module is applied to the audio signal after capture and before sending + * to the audio encoder. + */ + public void setCapturePostProcessing(@Nullable AudioProcessing processing) { + checkExternalAudioProcessorExists(); + long newPtr = nativeSetCapturePostProcessing(processing); + if (capturePostProcessingPtr != 0) { + JniCommon.nativeReleaseRef(capturePostProcessingPtr); + capturePostProcessingPtr = 0; + } + capturePostProcessingPtr = newPtr; + } + + /** + * Sets the render pre processing module. + * This module is applied to the audio signal after receiving from the audio + * decoder and before rendering. + */ + public void setRenderPreProcessing(@Nullable AudioProcessing processing) { + checkExternalAudioProcessorExists(); + long newPtr = nativeSetRenderPreProcessing(processing); + if (renderPreProcessingPtr != 0) { + JniCommon.nativeReleaseRef(renderPreProcessingPtr); + renderPreProcessingPtr = 0; + } + renderPreProcessingPtr = newPtr; + } + + /** + * Sets the bypass flag for the capture post processing module. + * If true, the registered audio processing will be bypassed. + */ + public void setBypassFlagForCapturePost( boolean bypass) { + checkExternalAudioProcessorExists(); + nativeSetBypassFlagForCapturePost(bypass); + } + + /** + * Sets the bypass flag for the render pre processing module. + * If true, the registered audio processing will be bypassed. + */ + public void setBypassFlagForRenderPre( boolean bypass) { + checkExternalAudioProcessorExists(); + nativeSetBypassFlagForRenderPre(bypass); + } + + /** + * Destroys the ExternalAudioProcessor. + */ + public void destroy() { + checkExternalAudioProcessorExists(); + if (renderPreProcessingPtr != 0) { + JniCommon.nativeReleaseRef(renderPreProcessingPtr); + renderPreProcessingPtr = 0; + } + if (capturePostProcessingPtr != 0) { + JniCommon.nativeReleaseRef(capturePostProcessingPtr); + capturePostProcessingPtr = 0; + } + nativeDestroy(); + apmPtr = 0; + } + + private void checkExternalAudioProcessorExists() { + if (apmPtr == 0) { + throw new IllegalStateException("ExternalAudioProcessor has been disposed."); + } + } + + private static native long nativeGetDefaultApm(); + private static native long nativeSetCapturePostProcessing(AudioProcessing processing); + private static native long nativeSetRenderPreProcessing(AudioProcessing processing); + private static native void nativeSetBypassFlagForCapturePost(boolean bypass); + private static native void nativeSetBypassFlagForRenderPre(boolean bypass); + private static native void nativeDestroy(); +} diff --git a/sdk/android/api/org/webrtc/FrameCryptor.java b/sdk/android/api/org/webrtc/FrameCryptor.java new file mode 100644 index 0000000000..d633e05005 --- /dev/null +++ b/sdk/android/api/org/webrtc/FrameCryptor.java @@ -0,0 +1,108 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; + +public class FrameCryptor { + public enum FrameCryptionState { + NEW, + OK, + ENCRYPTIONFAILED, + DECRYPTIONFAILED, + MISSINGKEY, + KEYRATCHETED, + INTERNALERROR; + + @CalledByNative("FrameCryptionState") + static FrameCryptionState fromNativeIndex(int nativeIndex) { + return values()[nativeIndex]; + } + } + + public static interface Observer { + @CalledByNative("Observer") + void onFrameCryptionStateChanged(String participantId, FrameCryptionState newState); + } + + private long nativeFrameCryptor; + private long observerPtr; + + public long getNativeFrameCryptor() { + return nativeFrameCryptor; + } + + @CalledByNative + public FrameCryptor(long nativeFrameCryptor) { + this.nativeFrameCryptor = nativeFrameCryptor; + this.observerPtr = 0; + } + + public void setEnabled(boolean enabled) { + checkFrameCryptorExists(); + nativeSetEnabled(nativeFrameCryptor, enabled); + } + + public boolean isEnabled() { + checkFrameCryptorExists(); + return nativeIsEnabled(nativeFrameCryptor); + } + + public int getKeyIndex() { + checkFrameCryptorExists(); + return nativeGetKeyIndex(nativeFrameCryptor); + } + + public void setKeyIndex(int index) { + checkFrameCryptorExists(); + nativeSetKeyIndex(nativeFrameCryptor, index); + } + + public void dispose() { + checkFrameCryptorExists(); + nativeUnSetObserver(nativeFrameCryptor); + JniCommon.nativeReleaseRef(nativeFrameCryptor); + nativeFrameCryptor = 0; + if (observerPtr != 0) { + JniCommon.nativeReleaseRef(observerPtr); + observerPtr = 0; + } + } + + public void setObserver(@Nullable Observer observer) { + checkFrameCryptorExists(); + long newPtr = nativeSetObserver(nativeFrameCryptor, observer); + if (observerPtr != 0) { + JniCommon.nativeReleaseRef(observerPtr); + observerPtr = 0; + } + newPtr = observerPtr; + } + + private void checkFrameCryptorExists() { + if (nativeFrameCryptor == 0) { + throw new IllegalStateException("FrameCryptor has been disposed."); + } + } + + private static native void nativeSetEnabled(long frameCryptorPointer, boolean enabled); + private static native boolean nativeIsEnabled(long frameCryptorPointer); + private static native void nativeSetKeyIndex(long frameCryptorPointer, int index); + private static native int nativeGetKeyIndex(long frameCryptorPointer); + private static native long nativeSetObserver(long frameCryptorPointer, Observer observer); + private static native void nativeUnSetObserver(long frameCryptorPointer); +} diff --git a/sdk/android/api/org/webrtc/FrameCryptorAlgorithm.java b/sdk/android/api/org/webrtc/FrameCryptorAlgorithm.java new file mode 100644 index 0000000000..d0d4dc8374 --- /dev/null +++ b/sdk/android/api/org/webrtc/FrameCryptorAlgorithm.java @@ -0,0 +1,22 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +public enum FrameCryptorAlgorithm { + AES_GCM, + AES_CBC, +} \ No newline at end of file diff --git a/sdk/android/api/org/webrtc/FrameCryptorFactory.java b/sdk/android/api/org/webrtc/FrameCryptorFactory.java new file mode 100644 index 0000000000..865a4b78bb --- /dev/null +++ b/sdk/android/api/org/webrtc/FrameCryptorFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +public class FrameCryptorFactory { + public static FrameCryptorKeyProvider createFrameCryptorKeyProvider( + boolean sharedKey, byte[] ratchetSalt, int ratchetWindowSize, byte[] uncryptedMagicBytes, int failureTolerance, int keyRingSize, boolean discardFrameWhenCryptorNotReady) { + return nativeCreateFrameCryptorKeyProvider(sharedKey, ratchetSalt, ratchetWindowSize, uncryptedMagicBytes, failureTolerance, keyRingSize, discardFrameWhenCryptorNotReady); + } + + public static FrameCryptor createFrameCryptorForRtpSender(PeerConnectionFactory factory, RtpSender rtpSender, + String participantId, FrameCryptorAlgorithm algorithm, FrameCryptorKeyProvider keyProvider) { + return nativeCreateFrameCryptorForRtpSender(factory.getNativeOwnedFactoryAndThreads(),rtpSender.getNativeRtpSender(), participantId, + algorithm.ordinal(), keyProvider.getNativeKeyProvider()); + } + + public static FrameCryptor createFrameCryptorForRtpReceiver(PeerConnectionFactory factory, RtpReceiver rtpReceiver, + String participantId, FrameCryptorAlgorithm algorithm, FrameCryptorKeyProvider keyProvider) { + return nativeCreateFrameCryptorForRtpReceiver(factory.getNativeOwnedFactoryAndThreads(), rtpReceiver.getNativeRtpReceiver(), participantId, + algorithm.ordinal(), keyProvider.getNativeKeyProvider()); + } + + private static native FrameCryptor nativeCreateFrameCryptorForRtpSender(long factory, + long rtpSender, String participantId, int algorithm, long nativeFrameCryptorKeyProvider); + private static native FrameCryptor nativeCreateFrameCryptorForRtpReceiver(long factory, + long rtpReceiver, String participantId, int algorithm, long nativeFrameCryptorKeyProvider); + + private static native FrameCryptorKeyProvider nativeCreateFrameCryptorKeyProvider( + boolean sharedKey, byte[] ratchetSalt, int ratchetWindowSize, byte[] uncryptedMagicBytes, int failureTolerance, int keyRingSize, boolean discardFrameWhenCryptorNotReady); +} diff --git a/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java new file mode 100644 index 0000000000..6ab0cdddf5 --- /dev/null +++ b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java @@ -0,0 +1,93 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +import java.util.ArrayList; + +public class FrameCryptorKeyProvider { + private long nativeKeyProvider; + + @CalledByNative + public FrameCryptorKeyProvider(long nativeKeyProvider) { + this.nativeKeyProvider = nativeKeyProvider; + } + + public long getNativeKeyProvider() { + return nativeKeyProvider; + } + + public boolean setSharedKey(int index, byte[] key) { + checkKeyProviderExists(); + return nativeSetSharedKey(nativeKeyProvider,index, key); + } + + public byte[] ratchetSharedKey(int index) { + checkKeyProviderExists(); + return nativeRatchetSharedKey(nativeKeyProvider, index); + } + + public byte[] exportSharedKey(int index) { + checkKeyProviderExists(); + return nativeExportSharedKey(nativeKeyProvider, index); + } + + public boolean setKey(String participantId, int index, byte[] key) { + checkKeyProviderExists(); + return nativeSetKey(nativeKeyProvider, participantId, index, key); + } + + public byte[] ratchetKey(String participantId, int index) { + checkKeyProviderExists(); + return nativeRatchetKey(nativeKeyProvider, participantId, index); + } + + public byte[] exportKey(String participantId, int index) { + checkKeyProviderExists(); + return nativeExportKey(nativeKeyProvider, participantId, index); + } + + public void setSifTrailer(byte[] sifTrailer) { + checkKeyProviderExists(); + nativeSetSifTrailer(nativeKeyProvider, sifTrailer); + } + + public void dispose() { + checkKeyProviderExists(); + JniCommon.nativeReleaseRef(nativeKeyProvider); + nativeKeyProvider = 0; + } + + private void checkKeyProviderExists() { + if (nativeKeyProvider == 0) { + throw new IllegalStateException("FrameCryptorKeyProvider has been disposed."); + } + } + private static native boolean nativeSetSharedKey( + long keyProviderPointer, int index, byte[] key); + private static native byte[] nativeRatchetSharedKey( + long keyProviderPointer, int index); + private static native byte[] nativeExportSharedKey( + long keyProviderPointer, int index); + private static native boolean nativeSetKey( + long keyProviderPointer, String participantId, int index, byte[] key); + private static native byte[] nativeRatchetKey( + long keyProviderPointer, String participantId, int index); + private static native byte[] nativeExportKey( + long keyProviderPointer, String participantId, int index); + private static native void nativeSetSifTrailer( + long keyProviderPointer, byte[] sifTrailer); +} \ No newline at end of file diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java index d43fc27fa0..05c39a4b38 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java @@ -143,11 +143,11 @@ public VideoCodecInfo[] getSupportedCodecs() { // supported by the decoder. if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) { supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>())); } supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>())); } } diff --git a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java index 340586ef81..9e80541969 100644 --- a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java +++ b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java @@ -9,6 +9,7 @@ */ package org.webrtc; +import java.util.List; public class LibaomAv1Encoder extends WrappedNativeVideoEncoder { @Override @@ -22,4 +23,10 @@ public long createNative(long webrtcEnvRef) { public boolean isHardwareEncoder() { return false; } + + static List scalabilityModes() { + return nativeGetSupportedScalabilityModes(); + } + + static native List nativeGetSupportedScalabilityModes(); } diff --git a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java index 71bcd1374b..c864ac2794 100644 --- a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java +++ b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java @@ -9,6 +9,7 @@ */ package org.webrtc; +import java.util.List; public class LibvpxVp9Encoder extends WrappedNativeVideoEncoder { @Override @@ -24,4 +25,10 @@ public boolean isHardwareEncoder() { } static native boolean nativeIsSupported(); + + static List scalabilityModes() { + return nativeGetSupportedScalabilityModes(); + } + + static native List nativeGetSupportedScalabilityModes(); } diff --git a/sdk/android/api/org/webrtc/PeerConnection.java b/sdk/android/api/org/webrtc/PeerConnection.java index d530bc2c0c..576e38c9e5 100644 --- a/sdk/android/api/org/webrtc/PeerConnection.java +++ b/sdk/android/api/org/webrtc/PeerConnection.java @@ -566,6 +566,17 @@ public static class RTCConfiguration { * See: https://www.chromestatus.com/feature/6269234631933952 */ public boolean offerExtmapAllowMixed; + + /** + * When this flag is set, ports not bound to any specific network interface + * will be used, in addition to normal ports bound to the enumerated + * interfaces. Without this flag, these "any address" ports would only be + * used when network enumeration fails or is disabled. But under certain + * conditions, these ports may succeed where others fail, so they may allow + * the application to work in a wider variety of environments, at the expense + * of having to allocate additional candidates. + */ + public boolean enableIceGatheringOnAnyAddressPorts; // TODO(deadbeef): Instead of duplicating the defaults here, we should do // something to pick up the defaults from C++. The Objective-C equivalent @@ -608,6 +619,7 @@ public RTCConfiguration(List iceServers) { turnLoggingId = null; enableImplicitRollback = false; offerExtmapAllowMixed = true; + enableIceGatheringOnAnyAddressPorts = false; } @CalledByNative("RTCConfiguration") @@ -816,6 +828,11 @@ boolean getEnableImplicitRollback() { boolean getOfferExtmapAllowMixed() { return offerExtmapAllowMixed; } + + @CalledByNative("RTCConfiguration") + boolean getEnableIceGatheringOnAnyAddressPorts() { + return enableIceGatheringOnAnyAddressPorts; + } }; private final List localStreams = new ArrayList<>(); diff --git a/sdk/android/api/org/webrtc/PeerConnectionFactory.java b/sdk/android/api/org/webrtc/PeerConnectionFactory.java index c46718fdd6..762ad3b678 100644 --- a/sdk/android/api/org/webrtc/PeerConnectionFactory.java +++ b/sdk/android/api/org/webrtc/PeerConnectionFactory.java @@ -20,6 +20,7 @@ import org.webrtc.RtpCapabilities; import org.webrtc.audio.AudioDeviceModule; import org.webrtc.audio.JavaAudioDeviceModule; +import org.webrtc.RtpCapabilities; /** * Java wrapper for a C++ PeerConnectionFactoryInterface. Main entry point to diff --git a/sdk/android/api/org/webrtc/RtpParameters.java b/sdk/android/api/org/webrtc/RtpParameters.java index 01f8082c57..4e3f106785 100644 --- a/sdk/android/api/org/webrtc/RtpParameters.java +++ b/sdk/android/api/org/webrtc/RtpParameters.java @@ -76,6 +76,7 @@ public static class Encoding { // If non-null, scale the width and height down by this factor for video. If null, // implementation default scaling factor will be used. @Nullable public Double scaleResolutionDownBy; + // Scalability modes are used to represent simulcast and SVC layers. @Nullable public String scalabilityMode; // SSRC to be used by this encoding. // Can't be changed between getParameters/setParameters. @@ -94,8 +95,8 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) { @CalledByNative("Encoding") Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority, Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate, - Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode, Long ssrc, - boolean adaptiveAudioPacketTime) { + Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode, + Long ssrc, boolean adaptiveAudioPacketTime) { this.rid = rid; this.active = active; this.bitratePriority = bitratePriority; diff --git a/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java b/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java index da39a1b6a7..306cbe57d8 100644 --- a/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java +++ b/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java @@ -2,7 +2,7 @@ public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder { - static native long nativeCreateEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info); + static native long nativeCreateEncoder(long webrtcEnvRef, VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info); VideoEncoderFactory primary; VideoEncoderFactory fallback; @@ -15,8 +15,8 @@ public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fa } @Override - public long createNativeVideoEncoder() { - return nativeCreateEncoder(primary, fallback, info); + public long createNative(long webrtcEnvRef) { + return nativeCreateEncoder(webrtcEnvRef, primary, fallback, info); } @Override @@ -25,3 +25,4 @@ public boolean isHardwareEncoder() { } } + diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java index 86d67d6d5c..e0f5153d47 100644 --- a/sdk/android/api/org/webrtc/VideoCodecInfo.java +++ b/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -14,6 +14,8 @@ import java.util.Arrays; import java.util.Locale; import java.util.Map; +import java.util.List; +import java.util.ArrayList; /** * Represent a video codec as encoded in SDP. @@ -34,15 +36,16 @@ public class VideoCodecInfo { public final String name; public final Map params; - public int[] scalabilityModes; + public final List scalabilityModes; + @Deprecated public final int payload; @CalledByNative - public VideoCodecInfo(String name, Map params) { + public VideoCodecInfo(String name, Map params, List scalabilityModes) { this.payload = 0; this.name = name; this.params = params; - this.scalabilityModes = new int[0]; + this.scalabilityModes = scalabilityModes; } @Deprecated @@ -50,7 +53,7 @@ public VideoCodecInfo(int payload, String name, Map params) { this.payload = payload; this.name = name; this.params = params; - this.scalabilityModes = new int[0]; + this.scalabilityModes = new ArrayList<>(); } @Override @@ -88,14 +91,7 @@ Map getParams() { } @CalledByNative - int[] getScalabilityModes() { + List getScalabilityModes() { return scalabilityModes; } - - @CalledByNative - void setScalabilityModes(int[] values) { - scalabilityModes = values; - } - - } diff --git a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java index 8515f48e3a..0bbc681e32 100644 --- a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java @@ -42,6 +42,7 @@ public static class Builder { private AudioTrackErrorCallback audioTrackErrorCallback; private AudioRecordErrorCallback audioRecordErrorCallback; private SamplesReadyCallback samplesReadyCallback; + private PlaybackSamplesReadyCallback playbackSamplesReadyCallback; private AudioTrackStateCallback audioTrackStateCallback; private AudioRecordStateCallback audioRecordStateCallback; private boolean useHardwareAcousticEchoCanceler = isBuiltInAcousticEchoCancelerSupported(); @@ -141,6 +142,14 @@ public Builder setSamplesReadyCallback(SamplesReadyCallback samplesReadyCallback return this; } + /** + * Set a callback to listen to the audio output passed to the AudioTrack. + */ + public Builder setPlaybackSamplesReadyCallback(PlaybackSamplesReadyCallback playbackSamplesReadyCallback) { + this.playbackSamplesReadyCallback = playbackSamplesReadyCallback; + return this; + } + /** * Set a callback to retrieve information from the AudioTrack on when audio starts and stop. */ @@ -269,7 +278,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() { samplesReadyCallback, audioRecordDataCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); final WebRtcAudioTrack audioOutput = new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback, - audioTrackStateCallback, useLowLatency, enableVolumeLogger); + audioTrackStateCallback, playbackSamplesReadyCallback, useLowLatency, enableVolumeLogger); return new JavaAudioDeviceModule(context, audioManager, audioInput, audioOutput, inputSampleRate, outputSampleRate, useStereoInput, useStereoOutput); } @@ -336,6 +345,11 @@ public static interface SamplesReadyCallback { void onWebRtcAudioRecordSamplesReady(AudioSamples samples); } + /** Called when new audio samples are ready. This should only be set for debug purposes */ + public static interface PlaybackSamplesReadyCallback { + void onWebRtcAudioTrackSamplesReady(AudioSamples samples); + } + /* AudioTrack */ // Audio playout/track error handler functions. public enum AudioTrackStartErrorCode { @@ -373,8 +387,8 @@ public static boolean isBuiltInNoiseSuppressorSupported() { private final Context context; private final AudioManager audioManager; - private final WebRtcAudioRecord audioInput; - private final WebRtcAudioTrack audioOutput; + public final WebRtcAudioRecord audioInput; + public final WebRtcAudioTrack audioOutput; private final int inputSampleRate; private final int outputSampleRate; private final boolean useStereoInput; diff --git a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java index 6f448124e8..d9fadabfd9 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java @@ -48,7 +48,7 @@ public AndroidVideoDecoderInstrumentationTest(String codecName, boolean useEglCo if (codecName.equals("H264")) { this.codecType = H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC; } else { - this.codecType = new VideoCodecInfo(codecName, new HashMap<>()); + this.codecType = new VideoCodecInfo(codecName, new HashMap<>(), new ArrayList<>()); } this.useEglContext = useEglContext; } diff --git a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java index fe608c794e..1a9dd5fc38 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java @@ -47,7 +47,7 @@ public void setUp() { @SmallTest @Test public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() { - VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>()); + VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>(), new ArrayList<>()); VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder); DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory); VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs(); @@ -62,7 +62,7 @@ public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() { public void getSupportedCodecs_hwVp8WithDifferentParams_twoVp8() { VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap() { { put("param", "value"); } - }); + }, new ArrayList<>()); VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder); DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory); VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs(); diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java index 8a5d9788ee..8be15624da 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java @@ -14,6 +14,7 @@ import androidx.annotation.Nullable; import androidx.test.filters.SmallTest; +import java.util.ArrayList; import java.util.HashMap; import org.junit.Before; import org.junit.Test; @@ -55,7 +56,7 @@ public void createDecoder_supportedCodec_returnsNotNull() { @Test public void createDecoder_unsupportedCodec_returnsNull() { VideoDecoderFactory factory = new SoftwareVideoDecoderFactory(); - VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap()); + VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap(), new ArrayList<>()); VideoDecoder decoder = factory.createDecoder(codec); assertThat(decoder).isNull(); } diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java index 696b423cde..0fa4c4cc17 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java @@ -14,6 +14,7 @@ import androidx.annotation.Nullable; import androidx.test.filters.SmallTest; +import java.util.ArrayList; import java.util.HashMap; import org.junit.Before; import org.junit.Test; @@ -52,7 +53,7 @@ public void createEncoder_supportedCodec_returnsNotNull() { @Test public void createEncoder_unsupportedCodec_returnsNull() { VideoEncoderFactory factory = new SoftwareVideoEncoderFactory(); - VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap()); + VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap(), new ArrayList<>()); VideoEncoder encoder = factory.createEncoder(codec); assertThat(encoder).isNull(); } diff --git a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java index 70151d3b78..72c5c64191 100644 --- a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java +++ b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.ArrayList; public class CodecsWrapperTestHelper { @CalledByNative @@ -20,7 +21,7 @@ public static VideoCodecInfo createTestVideoCodecInfo() { params.put( VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID, VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1); - VideoCodecInfo codec_info = new VideoCodecInfo("H264", params); + VideoCodecInfo codec_info = new VideoCodecInfo("H264", params, new ArrayList<>()); return codec_info; } diff --git a/sdk/android/src/java/org/webrtc/H264Utils.java b/sdk/android/src/java/org/webrtc/H264Utils.java index abb79c6582..4bf292ee12 100644 --- a/sdk/android/src/java/org/webrtc/H264Utils.java +++ b/sdk/android/src/java/org/webrtc/H264Utils.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.HashMap; +import java.util.ArrayList; /** Container for static helper functions related to dealing with H264 codecs. */ class H264Utils { @@ -38,9 +39,9 @@ public static Map getDefaultH264Params(boolean isHighProfile) { } public static VideoCodecInfo DEFAULT_H264_BASELINE_PROFILE_CODEC = - new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ false)); + new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ false), new ArrayList<>()); public static VideoCodecInfo DEFAULT_H264_HIGH_PROFILE_CODEC = - new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ true)); + new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ true), new ArrayList<>()); public static boolean isSameH264Profile( Map params1, Map params2) { diff --git a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java index 9bf6805094..66c3ec2600 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java @@ -73,11 +73,11 @@ public VideoCodecInfo[] getSupportedCodecs() { String name = type.name(); if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) { supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>())); } supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>())); } } diff --git a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java index 2b34e34013..25d10e4f61 100644 --- a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java +++ b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java @@ -20,12 +20,14 @@ import android.os.Process; import androidx.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.Arrays; import org.webrtc.CalledByNative; import org.webrtc.Logging; import org.webrtc.ThreadUtils; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback; +import org.webrtc.audio.JavaAudioDeviceModule.PlaybackSamplesReadyCallback; import org.webrtc.audio.LowLatencyAudioBufferManager; class WebRtcAudioTrack { @@ -62,7 +64,7 @@ class WebRtcAudioTrack { private ByteBuffer byteBuffer; - private @Nullable final AudioAttributes audioAttributes; + public @Nullable AudioAttributes audioAttributes; private @Nullable AudioTrack audioTrack; private @Nullable AudioTrackThread audioThread; private final VolumeLogger volumeLogger; @@ -76,6 +78,7 @@ class WebRtcAudioTrack { private final @Nullable AudioTrackErrorCallback errorCallback; private final @Nullable AudioTrackStateCallback stateCallback; + private final @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback; /** * Audio thread which keeps calling AudioTrack.write() to stream audio. @@ -129,6 +132,17 @@ public void run() { reportWebRtcAudioTrackError("AudioTrack.write failed: " + bytesWritten); } } + + if (audioSamplesReadyCallback != null && keepAlive) { + // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily + // at index 0. + byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), + sizeInBytes + byteBuffer.arrayOffset()); + audioSamplesReadyCallback.onWebRtcAudioTrackSamplesReady( + new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(), + audioTrack.getChannelCount(), audioTrack.getSampleRate(), data)); + } + if (useLowLatency) { bufferManager.maybeAdjustBufferSize(audioTrack); } @@ -154,13 +168,13 @@ public void stopThread() { @CalledByNative WebRtcAudioTrack(Context context, AudioManager audioManager) { this(context, audioManager, null /* audioAttributes */, null /* errorCallback */, - null /* stateCallback */, false /* useLowLatency */, true /* enableVolumeLogger */); + null /* stateCallback */, null /* audioSamplesReadyCallback */, false /* useLowLatency */, true /* enableVolumeLogger */); } WebRtcAudioTrack(Context context, AudioManager audioManager, @Nullable AudioAttributes audioAttributes, @Nullable AudioTrackErrorCallback errorCallback, - @Nullable AudioTrackStateCallback stateCallback, boolean useLowLatency, - boolean enableVolumeLogger) { + @Nullable AudioTrackStateCallback stateCallback, @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback, + boolean useLowLatency, boolean enableVolumeLogger) { threadChecker.detachThread(); this.context = context; this.audioManager = audioManager; @@ -168,6 +182,7 @@ public void stopThread() { this.errorCallback = errorCallback; this.stateCallback = stateCallback; this.volumeLogger = enableVolumeLogger ? new VolumeLogger(audioManager) : null; + this.audioSamplesReadyCallback = audioSamplesReadyCallback; this.useLowLatency = useLowLatency; Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); } diff --git a/sdk/android/src/jni/libaom_av1_encoder.cc b/sdk/android/src/jni/libaom_av1_encoder.cc index 9ed4abf5d6..17f7655c53 100644 --- a/sdk/android/src/jni/libaom_av1_encoder.cc +++ b/sdk/android/src/jni/libaom_av1_encoder.cc @@ -16,6 +16,9 @@ #include "sdk/android/generated_libaom_av1_encoder_jni/LibaomAv1Encoder_jni.h" #include "sdk/android/src/jni/jni_helpers.h" +#include +#include + namespace webrtc { namespace jni { @@ -26,5 +29,14 @@ jlong JNI_LibaomAv1Encoder_Create(JNIEnv* jni, jlong j_webrtc_env_ref) { .release()); } +static webrtc::ScopedJavaLocalRef JNI_LibaomAv1Encoder_GetSupportedScalabilityModes(JNIEnv* jni) { + std::vector modes; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + modes.push_back(std::string(webrtc::ScalabilityModeToString(scalability_mode))); + } + } + return NativeToJavaStringArray(jni, modes); +} } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/pc/audio_sink.cc b/sdk/android/src/jni/pc/audio_sink.cc new file mode 100644 index 0000000000..5bd88c75f6 --- /dev/null +++ b/sdk/android/src/jni/pc/audio_sink.cc @@ -0,0 +1,39 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/android/src/jni/pc/audio_sink.h" + +#include "sdk/android/generated_peerconnection_jni/AudioTrackSink_jni.h" + +namespace webrtc { +namespace jni { + +AudioTrackSinkWrapper::AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef& j_sink) + : j_sink_(jni, j_sink) {} + +AudioTrackSinkWrapper::~AudioTrackSinkWrapper() {} + +void AudioTrackSinkWrapper::OnData( + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + int length = (bits_per_sample / 8) * number_of_channels * number_of_frames; + ScopedJavaLocalRef audio_buffer = + NewDirectByteBuffer(jni, (void *) audio_data, length); + Java_AudioTrackSink_onData(jni, j_sink_, + audio_buffer, bits_per_sample, sample_rate, (int) number_of_channels, (int) number_of_frames, (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0)); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/audio_sink.h b/sdk/android/src/jni/pc/audio_sink.h new file mode 100644 index 0000000000..809f460e0c --- /dev/null +++ b/sdk/android/src/jni/pc/audio_sink.h @@ -0,0 +1,41 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ +#define SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ + +#include + +#include "api/media_stream_interface.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +class AudioTrackSinkWrapper : public webrtc::AudioTrackSinkInterface { + public: + AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef& j_sink); + ~AudioTrackSinkWrapper() override; + + private: + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override; + + const ScopedJavaGlobalRef j_sink_; +}; + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ diff --git a/sdk/android/src/jni/pc/audio_track.cc b/sdk/android/src/jni/pc/audio_track.cc index 36ed43f1d4..df2d605893 100644 --- a/sdk/android/src/jni/pc/audio_track.cc +++ b/sdk/android/src/jni/pc/audio_track.cc @@ -9,6 +9,8 @@ */ #include "api/media_stream_interface.h" +#include "sdk/android/src/jni/pc/audio_sink.h" + #include "sdk/android/generated_peerconnection_jni/AudioTrack_jni.h" namespace webrtc { @@ -20,5 +22,29 @@ static void JNI_AudioTrack_SetVolume(JNIEnv*, jlong j_p, jdouble volume) { source->SetVolume(volume); } +static void JNI_AudioTrack_AddSink(JNIEnv* jni, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) + ->AddSink(reinterpret_cast(j_native_sink)); +} + +static void JNI_AudioTrack_RemoveSink(JNIEnv* jni, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) + ->RemoveSink(reinterpret_cast(j_native_sink)); +} + +static jlong JNI_AudioTrack_WrapSink(JNIEnv* jni, + const JavaParamRef& sink) { + return jlongFromPointer(new AudioTrackSinkWrapper(jni, sink)); +} + +static void JNI_AudioTrack_FreeSink(JNIEnv* jni, jlong j_native_sink) { + delete reinterpret_cast(j_native_sink); +} + + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processing_factory.cc b/sdk/android/src/jni/pc/external_audio_processing_factory.cc new file mode 100644 index 0000000000..3d7ee7a4d9 --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processing_factory.cc @@ -0,0 +1,143 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sdk/android/src/jni/pc/external_audio_processing_factory.h" + +#include +#include + +#include "api/make_ref_counted.h" +#include "rtc_base/ref_counted_object.h" +#include "sdk/android/generated_peerconnection_jni/ExternalAudioProcessingFactory_jni.h" +#include "sdk/android/native_api/jni/java_types.h" +#include "sdk/android/native_api/jni/scoped_java_ref.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/pc/external_audio_processor.h" + +namespace webrtc { +namespace jni { + +ExternalAudioProcessingJni::ExternalAudioProcessingJni( + JNIEnv* jni, + const JavaRef& j_processing) + : j_processing_global_(jni, j_processing) {} +ExternalAudioProcessingJni::~ExternalAudioProcessingJni() {} +void ExternalAudioProcessingJni::Initialize(int sample_rate_hz, + int num_channels) { + JNIEnv* env = AttachCurrentThreadIfNeeded(); + Java_AudioProcessing_initialize(env, j_processing_global_, sample_rate_hz, + num_channels); +} + +void ExternalAudioProcessingJni::Reset(int new_rate) { + JNIEnv* env = AttachCurrentThreadIfNeeded(); + Java_AudioProcessing_reset(env, j_processing_global_, new_rate); +} + +void ExternalAudioProcessingJni::Process(int num_bands, int num_frames, int buffer_size, float* buffer) { + JNIEnv* env = AttachCurrentThreadIfNeeded(); + ScopedJavaLocalRef audio_buffer = + NewDirectByteBuffer(env, (void*)buffer, buffer_size * sizeof(float)); + Java_AudioProcessing_process(env, j_processing_global_, num_bands, num_frames, audio_buffer); +} + +ExternalAudioProcessingFactory::ExternalAudioProcessingFactory() { + capture_post_processor_ = new ExternalAudioProcessor(); + std::unique_ptr capture_post_processor( + capture_post_processor_); + + render_pre_processor_ = new ExternalAudioProcessor(); + std::unique_ptr render_pre_processor( + render_pre_processor_); + + apm_ = webrtc::AudioProcessingBuilder() + .SetCapturePostProcessing(std::move(capture_post_processor)) + .SetRenderPreProcessing(std::move(render_pre_processor)) + .Create(); + + webrtc::AudioProcessing::Config config; + apm_->ApplyConfig(config); +} + +static ExternalAudioProcessingFactory* default_processor_ptr; + +static jlong JNI_ExternalAudioProcessingFactory_GetDefaultApm(JNIEnv* env) { + if (!default_processor_ptr) { + auto default_processor = rtc::make_ref_counted(); + default_processor_ptr = default_processor.release(); + } + return webrtc::jni::jlongFromPointer(default_processor_ptr->apm().get()); +} + +static jlong JNI_ExternalAudioProcessingFactory_SetCapturePostProcessing( + JNIEnv* env, + const JavaParamRef& j_processing) { + if (!default_processor_ptr) { + return 0; + } + auto processing = + rtc::make_ref_counted(env, j_processing); + processing->AddRef(); + default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing( + processing.get()); + return jlongFromPointer(processing.get()); +} + +static jlong JNI_ExternalAudioProcessingFactory_SetRenderPreProcessing( + JNIEnv* env, + const JavaParamRef& j_processing) { + if (!default_processor_ptr) { + return 0; + } + auto processing = + rtc::make_ref_counted(env, j_processing); + processing->AddRef(); + default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing( + processing.get()); + return jlongFromPointer(processing.get()); +} + +static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForCapturePost( + JNIEnv* env, + jboolean bypass) { + if (!default_processor_ptr) { + return; + } + default_processor_ptr->capture_post_processor()->SetBypassFlag(bypass); +} + +static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForRenderPre( + JNIEnv* env, + jboolean bypass) { + if (!default_processor_ptr) { + return; + } + default_processor_ptr->render_pre_processor()->SetBypassFlag(bypass); +} + +static void JNI_ExternalAudioProcessingFactory_Destroy(JNIEnv* env) { + if (!default_processor_ptr) { + return; + } + default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing( + nullptr); + default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing( + nullptr); + delete default_processor_ptr; +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processing_factory.h b/sdk/android/src/jni/pc/external_audio_processing_factory.h new file mode 100644 index 0000000000..5dfebe81fc --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processing_factory.h @@ -0,0 +1,68 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#define WEBRTC_APM_DEBUG_DUMP 0 + +#include "rtc_base/ref_counted_object.h" +#include "sdk/android/native_api/jni/scoped_java_ref.h" +#include "sdk/android/src/jni/pc/external_audio_processor.h" +#include "sdk/android/src/jni/pc/external_audio_processing_interface.h" + +namespace webrtc { +namespace jni { + +class ExternalAudioProcessingJni + : public webrtc::ExternalAudioProcessingInterface, + public rtc::RefCountInterface { + public: + ExternalAudioProcessingJni(JNIEnv* jni, const JavaRef& j_processing); + ~ExternalAudioProcessingJni(); + + protected: + virtual void Initialize(int sample_rate_hz, int num_channels) override; + virtual void Reset(int new_rate) override; + virtual void Process(int num_bans, int num_frames, int buffer_size, float* buffer) override; + + private: + const ScopedJavaGlobalRef j_processing_global_; + const ScopedJavaGlobalRef j_processing_; +}; + +class ExternalAudioProcessingFactory : public rtc::RefCountInterface { + public: + ExternalAudioProcessingFactory(); + virtual ~ExternalAudioProcessingFactory() = default; + + ExternalAudioProcessor* capture_post_processor() { + return capture_post_processor_; + } + + ExternalAudioProcessor* render_pre_processor() { + return render_pre_processor_; + } + + rtc::scoped_refptr apm() { return apm_; } + + private: + rtc::scoped_refptr apm_; + ExternalAudioProcessor* capture_post_processor_; + ExternalAudioProcessor* render_pre_processor_; +}; + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processing_interface.h b/sdk/android/src/jni/pc/external_audio_processing_interface.h new file mode 100644 index 0000000000..1202be106b --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processing_interface.h @@ -0,0 +1,33 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_ANDROID_JNI_EXTERNALAUDIOPROCESSORINTERFACE_H_ +#define API_ANDROID_JNI_EXTERNALAUDIOPROCESSORINTERFACE_H_ + +namespace webrtc { + +class ExternalAudioProcessingInterface { + public: + virtual void Initialize(int sample_rate_hz, int num_channels) = 0; + virtual void Reset(int new_rate) = 0; + virtual void Process(int num_bands, int num_frames, int buffer_size, float* buffer) = 0; + + protected: + virtual ~ExternalAudioProcessingInterface() = default; +}; + +} // namespace webrtc + +#endif // API_ANDROID_JNI_EXTERNALAUDIOPROCESSORINTERFACE_H_ diff --git a/sdk/android/src/jni/pc/external_audio_processor.cc b/sdk/android/src/jni/pc/external_audio_processor.cc new file mode 100644 index 0000000000..274982d6d4 --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processor.cc @@ -0,0 +1,72 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sdk/android/src/jni/pc/external_audio_processor.h" + +namespace webrtc { + +void ExternalAudioProcessor::SetExternalAudioProcessing( + ExternalAudioProcessingInterface* processor) { + webrtc::MutexLock lock(&mutex_); + external_processor_ = processor; + if (initialized_) { + external_processor_->Initialize(sample_rate_hz_, num_channels_); + } +} + +void ExternalAudioProcessor::SetBypassFlag(bool bypass) { + webrtc::MutexLock lock(&mutex_); + bypass_flag_ = bypass; +} + +void ExternalAudioProcessor::Initialize(int sample_rate_hz, int num_channels) { + webrtc::MutexLock lock(&mutex_); + sample_rate_hz_ = sample_rate_hz; + num_channels_ = num_channels; + if (external_processor_) { + external_processor_->Initialize(sample_rate_hz, num_channels); + } + initialized_ = true; +} + +void ExternalAudioProcessor::Process(webrtc::AudioBuffer* audio) { + webrtc::MutexLock lock(&mutex_); + if (!external_processor_ || bypass_flag_ || !initialized_) { + return; + } + + size_t num_frames = audio->num_frames(); + size_t num_bands =audio->num_bands(); + + // 1 buffer = 10ms of frames + int rate = num_frames * 100; + + if (rate != sample_rate_hz_) { + external_processor_->Reset(rate); + sample_rate_hz_ = rate; + } + + external_processor_->Process(num_bands, num_frames, kNsFrameSize * num_bands, audio->channels()[0]); +} + +std::string ExternalAudioProcessor::ToString() const { + return "ExternalAudioProcessor"; +} + +void ExternalAudioProcessor::SetRuntimeSetting( + webrtc::AudioProcessing::RuntimeSetting setting) {} + +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processor.h b/sdk/android/src/jni/pc/external_audio_processor.h new file mode 100644 index 0000000000..1dc31809fc --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processor.h @@ -0,0 +1,57 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_ANDROID_SRC_JNI_PC_EXTERNAL_AUDIO_PROCESSOR_H_ +#define SDK_ANDROID_SRC_JNI_PC_EXTERNAL_AUDIO_PROCESSOR_H_ + +#define WEBRTC_APM_DEBUG_DUMP 0 + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/audio_processing_impl.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "sdk/android/src/jni/pc/external_audio_processing_interface.h" + +namespace webrtc { + +class ExternalAudioProcessor : public webrtc::CustomProcessing { + public: + ExternalAudioProcessor() = default; + ~ExternalAudioProcessor() override = default; + + void SetExternalAudioProcessing( + ExternalAudioProcessingInterface* processor); + + void SetBypassFlag(bool bypass); + + private: + void Initialize(int sample_rate_hz, int num_channels) override; + void Process(webrtc::AudioBuffer* audio) override; + std::string ToString() const override; + void SetRuntimeSetting( + webrtc::AudioProcessing::RuntimeSetting setting) override; + + private: + mutable webrtc::Mutex mutex_; + ExternalAudioProcessingInterface* external_processor_; + bool bypass_flag_ = false; + bool initialized_ = false; + int sample_rate_hz_ = 0; + int num_channels_ = 0; +}; + +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_PC_EXTERNAL_AUDIO_PROCESSOR_H_ diff --git a/sdk/android/src/jni/pc/frame_cryptor.cc b/sdk/android/src/jni/pc/frame_cryptor.cc new file mode 100644 index 0000000000..af2fd8f2b0 --- /dev/null +++ b/sdk/android/src/jni/pc/frame_cryptor.cc @@ -0,0 +1,202 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sdk/android/src/jni/pc/frame_cryptor.h" + +#include "api/rtp_receiver_interface.h" +#include "api/rtp_sender_interface.h" +#include "rtc_base/ref_counted_object.h" +#include "sdk/android/generated_peerconnection_jni/FrameCryptorFactory_jni.h" +#include "sdk/android/generated_peerconnection_jni/FrameCryptor_jni.h" +#include "sdk/android/native_api/jni/java_types.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/pc/frame_cryptor_key_provider.h" +#include "sdk/android/src/jni/pc/owned_factory_and_threads.h" + +namespace webrtc { +namespace jni { + +FrameCryptorObserverJni::FrameCryptorObserverJni( + JNIEnv* jni, + const JavaRef& j_observer) + : j_observer_global_(jni, j_observer) {} + +FrameCryptorObserverJni::~FrameCryptorObserverJni() {} + +void FrameCryptorObserverJni::OnFrameCryptionStateChanged( + const std::string participant_id, + FrameCryptionState new_state) { + JNIEnv* env = AttachCurrentThreadIfNeeded(); + Java_Observer_onFrameCryptionStateChanged( + env, j_observer_global_, NativeToJavaString(env, participant_id), + Java_FrameCryptionState_fromNativeIndex(env, new_state)); +} + +ScopedJavaLocalRef NativeToJavaFrameCryptor( + JNIEnv* env, + rtc::scoped_refptr cryptor) { + if (!cryptor) + return nullptr; + // Sender is now owned by the Java object, and will be freed from + // FrameCryptor.dispose(). + return Java_FrameCryptor_Constructor(env, + jlongFromPointer(cryptor.release())); +} + +static void JNI_FrameCryptor_SetEnabled(JNIEnv* jni, + jlong j_frame_cryptor_pointer, + jboolean j_enabled) { + reinterpret_cast(j_frame_cryptor_pointer) + ->SetEnabled(j_enabled); +} + +static jboolean JNI_FrameCryptor_IsEnabled(JNIEnv* jni, + jlong j_frame_cryptor_pointer) { + return reinterpret_cast(j_frame_cryptor_pointer) + ->enabled(); +} + +static void JNI_FrameCryptor_SetKeyIndex(JNIEnv* jni, + jlong j_frame_cryptor_pointer, + jint j_index) { + reinterpret_cast(j_frame_cryptor_pointer) + ->SetKeyIndex(j_index); +} + +static jint JNI_FrameCryptor_GetKeyIndex(JNIEnv* jni, + jlong j_frame_cryptor_pointer) { + return reinterpret_cast(j_frame_cryptor_pointer) + ->key_index(); +} + +static jlong JNI_FrameCryptor_SetObserver( + JNIEnv* jni, + jlong j_frame_cryptor_pointer, + const JavaParamRef& j_observer) { + auto observer = + rtc::make_ref_counted(jni, j_observer); + observer->AddRef(); + reinterpret_cast(j_frame_cryptor_pointer) + ->RegisterFrameCryptorTransformerObserver(observer); + return jlongFromPointer(observer.get()); +} + +static void JNI_FrameCryptor_UnSetObserver(JNIEnv* jni, + jlong j_frame_cryptor_pointer) { + reinterpret_cast(j_frame_cryptor_pointer) + ->UnRegisterFrameCryptorTransformerObserver(); +} + +webrtc::FrameCryptorTransformer::Algorithm AlgorithmFromIndex(int index) { + switch (index) { + case 0: + return webrtc::FrameCryptorTransformer::Algorithm::kAesGcm; + case 1: + return webrtc::FrameCryptorTransformer::Algorithm::kAesCbc; + default: + return webrtc::FrameCryptorTransformer::Algorithm::kAesGcm; + } +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorFactory_CreateFrameCryptorForRtpReceiver( + JNIEnv* env, + jlong native_factory, + jlong j_rtp_receiver_pointer, + const base::android::JavaParamRef& participantId, + jint j_algorithm_index, + jlong j_key_provider) { + OwnedFactoryAndThreads* factory = + reinterpret_cast(native_factory); + auto keyProvider = + reinterpret_cast(j_key_provider); + auto participant_id = JavaToStdString(env, participantId); + auto rtpReceiver = + reinterpret_cast(j_rtp_receiver_pointer); + auto mediaType = + rtpReceiver->track()->kind() == "audio" + ? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame + : webrtc::FrameCryptorTransformer::MediaType::kVideoFrame; + auto frame_crypto_transformer = + rtc::scoped_refptr( + new webrtc::FrameCryptorTransformer(factory->signaling_thread(), + participant_id, mediaType, AlgorithmFromIndex(j_algorithm_index), + rtc::scoped_refptr(keyProvider))); + + rtpReceiver->SetDepacketizerToDecoderFrameTransformer( + frame_crypto_transformer); + frame_crypto_transformer->SetEnabled(false); + + return NativeToJavaFrameCryptor(env, frame_crypto_transformer); +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorFactory_CreateFrameCryptorForRtpSender( + JNIEnv* env, + jlong native_factory, + jlong j_rtp_sender_pointer, + const base::android::JavaParamRef& participantId, + jint j_algorithm_index, + jlong j_key_provider) { + OwnedFactoryAndThreads* factory = + reinterpret_cast(native_factory); + auto keyProvider = + reinterpret_cast(j_key_provider); + auto rtpSender = reinterpret_cast(j_rtp_sender_pointer); + auto participant_id = JavaToStdString(env, participantId); + auto mediaType = + rtpSender->track()->kind() == "audio" + ? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame + : webrtc::FrameCryptorTransformer::MediaType::kVideoFrame; + auto frame_crypto_transformer = + rtc::scoped_refptr( + new webrtc::FrameCryptorTransformer(factory->signaling_thread(), + participant_id, mediaType, AlgorithmFromIndex(j_algorithm_index), + rtc::scoped_refptr(keyProvider))); + + rtpSender->SetEncoderToPacketizerFrameTransformer(frame_crypto_transformer); + frame_crypto_transformer->SetEnabled(false); + + return NativeToJavaFrameCryptor(env, frame_crypto_transformer); +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorFactory_CreateFrameCryptorKeyProvider( + JNIEnv* env, + jboolean j_shared, + const base::android::JavaParamRef& j_ratchetSalt, + jint j_ratchetWindowSize, + const base::android::JavaParamRef& j_uncryptedMagicBytes, + jint j_failureTolerance, + jint j_keyRingSize, + jboolean j_discardFrameWhenCryptorNotReady) { + auto ratchetSalt = JavaToNativeByteArray(env, j_ratchetSalt); + KeyProviderOptions options; + options.ratchet_salt = + std::vector(ratchetSalt.begin(), ratchetSalt.end()); + options.ratchet_window_size = j_ratchetWindowSize; + auto uncryptedMagicBytes = JavaToNativeByteArray(env, j_uncryptedMagicBytes); + options.uncrypted_magic_bytes = + std::vector(uncryptedMagicBytes.begin(), uncryptedMagicBytes.end()); + options.shared_key = j_shared; + options.failure_tolerance = j_failureTolerance; + options.key_ring_size = j_keyRingSize; + options.discard_frame_when_cryptor_not_ready = j_discardFrameWhenCryptorNotReady; + return NativeToJavaFrameCryptorKeyProvider( + env, rtc::make_ref_counted(options)); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/frame_cryptor.h b/sdk/android/src/jni/pc/frame_cryptor.h new file mode 100644 index 0000000000..dd0788d212 --- /dev/null +++ b/sdk/android/src/jni/pc/frame_cryptor.h @@ -0,0 +1,49 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_ANDROID_SRC_JNI_PC_FRAME_CRYPTOR_H_ +#define SDK_ANDROID_SRC_JNI_PC_FRAME_CRYPTOR_H_ + +#include + +#include "api/crypto/frame_crypto_transformer.h" +#include "sdk/android/native_api/jni/scoped_java_ref.h" + +namespace webrtc { +namespace jni { + +ScopedJavaLocalRef NativeToJavaFrameCryptor( + JNIEnv* env, + rtc::scoped_refptr cryptor); + +class FrameCryptorObserverJni : public FrameCryptorTransformerObserver { + public: + FrameCryptorObserverJni(JNIEnv* jni, const JavaRef& j_observer); + ~FrameCryptorObserverJni() override; + + protected: + void OnFrameCryptionStateChanged(const std::string participant_id, + FrameCryptionState state) override; + + private: + const ScopedJavaGlobalRef j_observer_global_; + const ScopedJavaGlobalRef j_observer_; +}; + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_PC_FRAME_CRYPTOR_H_ diff --git a/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc new file mode 100644 index 0000000000..e41d16ed91 --- /dev/null +++ b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc @@ -0,0 +1,123 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sdk/android/src/jni/pc/frame_cryptor_key_provider.h" + +#include "sdk/android/generated_peerconnection_jni/FrameCryptorKeyProvider_jni.h" +#include "sdk/android/native_api/jni/java_types.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +ScopedJavaLocalRef NativeToJavaFrameCryptorKeyProvider( + JNIEnv* env, + rtc::scoped_refptr key_provider) { + if (!key_provider) + return nullptr; + // Sender is now owned by the Java object, and will be freed from + // FrameCryptorKeyProvider.dispose(). + return Java_FrameCryptorKeyProvider_Constructor( + env, jlongFromPointer(key_provider.release())); +} + +static jboolean JNI_FrameCryptorKeyProvider_SetSharedKey( + JNIEnv* jni, + jlong j_key_provider, + jint j_index, + const base::android::JavaParamRef& j_key) { + auto key = JavaToNativeByteArray(jni, j_key); + return reinterpret_cast(j_key_provider) + ->SetSharedKey(j_index,std::vector(key.begin(), key.end())); +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorKeyProvider_RatchetSharedKey( + JNIEnv* env, + jlong keyProviderPointer, + jint j_index) { + auto key_provider = + reinterpret_cast(keyProviderPointer); + auto newKey = key_provider->RatchetSharedKey(j_index); + std::vector int8tKey = + std::vector(newKey.begin(), newKey.end()); + return NativeToJavaByteArray(env, rtc::ArrayView(int8tKey)); +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorKeyProvider_ExportSharedKey( + JNIEnv* env, + jlong keyProviderPointer, + jint j_index) { + auto key_provider = + reinterpret_cast(keyProviderPointer); + auto key = key_provider->ExportSharedKey(j_index); + std::vector int8tKey = std::vector(key.begin(), key.end()); + return NativeToJavaByteArray(env, rtc::ArrayView(int8tKey)); +} + +static jboolean JNI_FrameCryptorKeyProvider_SetKey( + JNIEnv* jni, + jlong j_key_provider, + const base::android::JavaParamRef& participantId, + jint j_index, + const base::android::JavaParamRef& j_key) { + auto key = JavaToNativeByteArray(jni, j_key); + auto participant_id = JavaToStdString(jni, participantId); + return reinterpret_cast(j_key_provider) + ->SetKey(participant_id, j_index, + std::vector(key.begin(), key.end())); +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorKeyProvider_RatchetKey( + JNIEnv* env, + jlong keyProviderPointer, + const base::android::JavaParamRef& participantId, + jint j_index) { + auto participant_id = JavaToStdString(env, participantId); + auto key_provider = + reinterpret_cast(keyProviderPointer); + auto newKey = key_provider->RatchetKey(participant_id, j_index); + std::vector int8tKey = + std::vector(newKey.begin(), newKey.end()); + return NativeToJavaByteArray(env, rtc::ArrayView(int8tKey)); +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorKeyProvider_ExportKey( + JNIEnv* env, + jlong keyProviderPointer, + const base::android::JavaParamRef& participantId, + jint j_index) { + auto participant_id = JavaToStdString(env, participantId); + auto key_provider = + reinterpret_cast(keyProviderPointer); + auto key = key_provider->ExportKey(participant_id, j_index); + std::vector int8tKey = std::vector(key.begin(), key.end()); + return NativeToJavaByteArray(env, rtc::ArrayView(int8tKey)); +} + +static void JNI_FrameCryptorKeyProvider_SetSifTrailer( + JNIEnv* jni, + jlong j_key_provider, + const base::android::JavaParamRef& j_trailer) { + auto trailer = JavaToNativeByteArray(jni, j_trailer); + reinterpret_cast(j_key_provider) + ->SetSifTrailer(std::vector(trailer.begin(), trailer.end())); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/frame_cryptor_key_provider.h b/sdk/android/src/jni/pc/frame_cryptor_key_provider.h new file mode 100644 index 0000000000..8832a83035 --- /dev/null +++ b/sdk/android/src/jni/pc/frame_cryptor_key_provider.h @@ -0,0 +1,35 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_ANDROID_SRC_JNI_PC_FRAME_CRYPTOR_KEY_PROVIDER_H_ +#define SDK_ANDROID_SRC_JNI_PC_FRAME_CRYPTOR_KEY_PROVIDER_H_ + +#include + +#include "api/crypto/frame_crypto_transformer.h" +#include "sdk/android/native_api/jni/scoped_java_ref.h" + +namespace webrtc { +namespace jni { + +ScopedJavaLocalRef NativeToJavaFrameCryptorKeyProvider( + JNIEnv* env, + rtc::scoped_refptr cryptor); + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_PC_FRAME_CRYPTOR_KEY_PROVIDER_H_ diff --git a/sdk/android/src/jni/pc/peer_connection.cc b/sdk/android/src/jni/pc/peer_connection.cc index e5fb11e3b4..deab006bec 100644 --- a/sdk/android/src/jni/pc/peer_connection.cc +++ b/sdk/android/src/jni/pc/peer_connection.cc @@ -272,6 +272,9 @@ void JavaToNativeRTCConfiguration( rtc_config->enable_implicit_rollback = Java_RTCConfiguration_getEnableImplicitRollback(jni, j_rtc_config); + rtc_config->enable_any_address_ports = + Java_RTCConfiguration_getEnableIceGatheringOnAnyAddressPorts(jni, j_rtc_config); + ScopedJavaLocalRef j_turn_logging_id = Java_RTCConfiguration_getTurnLoggingId(jni, j_rtc_config); if (!IsNull(jni, j_turn_logging_id)) { diff --git a/sdk/android/src/jni/pc/peer_connection_factory.cc b/sdk/android/src/jni/pc/peer_connection_factory.cc index 9a21e10ede..c323113061 100644 --- a/sdk/android/src/jni/pc/peer_connection_factory.cc +++ b/sdk/android/src/jni/pc/peer_connection_factory.cc @@ -32,6 +32,7 @@ #include "sdk/android/src/jni/logging/log_sink.h" #include "sdk/android/src/jni/pc/android_network_monitor.h" #include "sdk/android/src/jni/pc/audio.h" +#include "sdk/android/src/jni/pc/rtp_capabilities.h" #include "sdk/android/src/jni/pc/ice_candidate.h" #include "sdk/android/src/jni/pc/media_stream_track.h" #include "sdk/android/src/jni/pc/owned_factory_and_threads.h" diff --git a/sdk/android/src/jni/pc/rtp_parameters.cc b/sdk/android/src/jni/pc/rtp_parameters.cc index dfc7f17f6d..6feb6a631b 100644 --- a/sdk/android/src/jni/pc/rtp_parameters.cc +++ b/sdk/android/src/jni/pc/rtp_parameters.cc @@ -120,7 +120,7 @@ RtpEncodingParameters JavaToNativeRtpEncodingParameters( ScopedJavaLocalRef j_scalability_mode = Java_Encoding_getScalabilityMode(jni, j_encoding_parameters); if (!IsNull(jni, j_scalability_mode)) { - encoding.scalability_mode = JavaToNativeString(jni, j_scalability_mode); + encoding.scalability_mode = JavaToNativeString(jni,j_scalability_mode); } encoding.adaptive_ptime = Java_Encoding_getAdaptivePTime(jni, j_encoding_parameters); diff --git a/sdk/android/src/jni/simulcast_video_encoder.cc b/sdk/android/src/jni/simulcast_video_encoder.cc index 18daf6f062..6874c1821b 100644 --- a/sdk/android/src/jni/simulcast_video_encoder.cc +++ b/sdk/android/src/jni/simulcast_video_encoder.cc @@ -15,7 +15,7 @@ extern "C" { #endif // (VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) -JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jobject primary, jobject fallback, jobject info) { +JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jlong webrtcEnvRef, jobject primary, jobject fallback, jobject info) { RTC_LOG(LS_INFO) << "Create simulcast video encoder"; JavaParamRef info_ref(info); SdpVideoFormat format = VideoCodecInfoToSdpVideoFormat(env, info_ref); @@ -23,8 +23,9 @@ JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncode // TODO: 影響は軽微だが、リークする可能性があるので将来的に修正したい // https://github.com/shiguredo-webrtc-build/webrtc-build/pull/16#pullrequestreview-600675795 return NativeToJavaPointer(std::make_unique( + *reinterpret_cast(webrtcEnvRef), JavaToNativeVideoEncoderFactory(env, primary).release(), - fallback != nullptr ? JavaToNativeVideoEncoderFactory(env, fallback).release() : nullptr, + JavaToNativeVideoEncoderFactory(env, fallback).release(), format).release()); } diff --git a/sdk/android/src/jni/simulcast_video_encoder.h b/sdk/android/src/jni/simulcast_video_encoder.h index 519be778e8..d80ccde014 100644 --- a/sdk/android/src/jni/simulcast_video_encoder.h +++ b/sdk/android/src/jni/simulcast_video_encoder.h @@ -14,7 +14,7 @@ extern "C" { */ JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder - (JNIEnv *, jclass, jobject, jobject, jobject); + (JNIEnv *, jclass, jlong, jobject, jobject, jobject); #ifdef __cplusplus } diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc index 42e7b5d897..342d98135e 100644 --- a/sdk/android/src/jni/video_codec_info.cc +++ b/sdk/android/src/jni/video_codec_info.cc @@ -13,6 +13,8 @@ #include "sdk/android/generated_video_jni/VideoCodecInfo_jni.h" #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/jni_helpers.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "absl/container/inlined_vector.h" @@ -21,15 +23,20 @@ namespace jni { SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni, const JavaRef& j_info) { - absl::InlinedVector scalabilityModes; - std::vector javaScalabilityModes = JavaToNativeIntArray(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info)); - for (const auto& scalabilityMode : javaScalabilityModes) { - scalabilityModes.push_back(static_cast(scalabilityMode)); + std::vector params = + JavaToStdVectorStrings(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info)); + absl::InlinedVector + scalability_modes; + for (auto mode : params) { + auto scalability_mode = ScalabilityModeFromString(mode); + if (scalability_mode != absl::nullopt) { + scalability_modes.push_back(*scalability_mode); + } } return SdpVideoFormat( JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info)), JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)), - scalabilityModes); + scalability_modes); } ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( @@ -37,18 +44,17 @@ ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( const SdpVideoFormat& format) { ScopedJavaLocalRef j_params = NativeToJavaStringMap(jni, format.parameters); - - ScopedJavaLocalRef codec = Java_VideoCodecInfo_Constructor( - jni, NativeToJavaString(jni, format.name), j_params); - - size_t size = format.scalability_modes.size(); - std::vector temp(size); - for (size_t i = 0; i < size; i++) { - temp[i] = static_cast(format.scalability_modes[i]); + webrtc::ScopedJavaLocalRef j_scalability_modes; + if (!format.scalability_modes.empty()) { + JavaListBuilder builder(jni); + for (auto mode : format.scalability_modes) { + std::string scalability_mode(ScalabilityModeToString(mode)); + builder.add(NativeToJavaString(jni, scalability_mode)); + } + j_scalability_modes = builder.java_list(); } - Java_VideoCodecInfo_setScalabilityModes(jni, codec, NativeToJavaIntArray(jni, temp)); - - return codec; + return Java_VideoCodecInfo_Constructor( + jni, NativeToJavaString(jni, format.name), j_params, j_scalability_modes); } } // namespace jni diff --git a/sdk/android/src/jni/vp9_codec.cc b/sdk/android/src/jni/vp9_codec.cc index dcce5bb5e8..9131b8306d 100644 --- a/sdk/android/src/jni/vp9_codec.cc +++ b/sdk/android/src/jni/vp9_codec.cc @@ -11,11 +11,18 @@ #include #include "api/environment/environment.h" +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "modules/video_coding/svc/create_scalability_structure.h" + #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Decoder_jni.h" #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Encoder_jni.h" #include "sdk/android/src/jni/jni_helpers.h" +#include +#include + namespace webrtc { namespace jni { @@ -37,5 +44,14 @@ static jboolean JNI_LibvpxVp9Decoder_IsSupported(JNIEnv* jni) { return !SupportedVP9Codecs().empty(); } +static webrtc::ScopedJavaLocalRef JNI_LibvpxVp9Encoder_GetSupportedScalabilityModes(JNIEnv* jni) { + std::vector modes; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + modes.push_back(std::string(webrtc::ScalabilityModeToString(scalability_mode))); + } + } + return NativeToJavaStringArray(jni, modes); +} } // namespace jni } // namespace webrtc diff --git a/sdk/objc/PrivacyInfo.xcprivacy b/sdk/objc/PrivacyInfo.xcprivacy new file mode 100644 index 0000000000..7204a67c33 --- /dev/null +++ b/sdk/objc/PrivacyInfo.xcprivacy @@ -0,0 +1,28 @@ + + + + + NSPrivacyCollectedDataTypes + + NSPrivacyAccessedAPITypes + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryFileTimestamp + NSPrivacyAccessedAPITypeReasons + + C617.1 + + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategorySystemBootTime + NSPrivacyAccessedAPITypeReasons + + 35F9.1 + 8FFB.1 + + + + + \ No newline at end of file diff --git a/sdk/objc/api/RTCVideoRendererAdapter+Private.h b/sdk/objc/api/RTCVideoRendererAdapter+Private.h index 9b123d2d05..cac9ab665c 100644 --- a/sdk/objc/api/RTCVideoRendererAdapter+Private.h +++ b/sdk/objc/api/RTCVideoRendererAdapter+Private.h @@ -16,7 +16,7 @@ NS_ASSUME_NONNULL_BEGIN -@interface RTCVideoRendererAdapter () +@interface RTC_OBJC_TYPE(RTCVideoRendererAdapter) () /** * The Objective-C video renderer passed to this adapter during construction. diff --git a/sdk/objc/api/RTCVideoRendererAdapter.h b/sdk/objc/api/RTCVideoRendererAdapter.h index b0b6f04488..bbb8c6e71c 100644 --- a/sdk/objc/api/RTCVideoRendererAdapter.h +++ b/sdk/objc/api/RTCVideoRendererAdapter.h @@ -10,6 +10,8 @@ #import +#import "RTCMacros.h" + NS_ASSUME_NONNULL_BEGIN /* @@ -18,7 +20,7 @@ NS_ASSUME_NONNULL_BEGIN * adapter adapts calls made to that interface to the RTCVideoRenderer supplied * during construction. */ -@interface RTCVideoRendererAdapter : NSObject +@interface RTC_OBJC_TYPE (RTCVideoRendererAdapter): NSObject - (instancetype)init NS_UNAVAILABLE; diff --git a/sdk/objc/api/RTCVideoRendererAdapter.mm b/sdk/objc/api/RTCVideoRendererAdapter.mm index ef02f72f60..d992c64108 100644 --- a/sdk/objc/api/RTCVideoRendererAdapter.mm +++ b/sdk/objc/api/RTCVideoRendererAdapter.mm @@ -17,10 +17,9 @@ namespace webrtc { -class VideoRendererAdapter - : public rtc::VideoSinkInterface { +class VideoRendererAdapter : public rtc::VideoSinkInterface { public: - VideoRendererAdapter(RTCVideoRendererAdapter* adapter) { + VideoRendererAdapter(RTC_OBJC_TYPE(RTCVideoRendererAdapter) * adapter) { adapter_ = adapter; size_ = CGSizeZero; } @@ -28,9 +27,9 @@ void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override { RTC_OBJC_TYPE(RTCVideoFrame)* videoFrame = NativeToObjCVideoFrame(nativeVideoFrame); - CGSize current_size = (videoFrame.rotation % 180 == 0) - ? CGSizeMake(videoFrame.width, videoFrame.height) - : CGSizeMake(videoFrame.height, videoFrame.width); + CGSize current_size = (videoFrame.rotation % 180 == 0) ? + CGSizeMake(videoFrame.width, videoFrame.height) : + CGSizeMake(videoFrame.height, videoFrame.width); if (!CGSizeEqualToSize(size_, current_size)) { size_ = current_size; @@ -40,12 +39,12 @@ void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override { } private: - __weak RTCVideoRendererAdapter *adapter_; + __weak RTC_OBJC_TYPE(RTCVideoRendererAdapter) * adapter_; CGSize size_; }; -} +} // namespace webrtc -@implementation RTCVideoRendererAdapter { +@implementation RTC_OBJC_TYPE (RTCVideoRendererAdapter) { std::unique_ptr _adapter; } @@ -60,7 +59,7 @@ - (instancetype)initWithNativeRenderer:(id)vide return self; } -- (rtc::VideoSinkInterface *)nativeVideoRenderer { +- (rtc::VideoSinkInterface*)nativeVideoRenderer { return _adapter.get(); } diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h index 4eb91b93c7..73c1a4e26a 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h @@ -21,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN -@interface RTCAudioDeviceModule () +@interface RTC_OBJC_TYPE(RTCAudioDeviceModule) () - (instancetype)initWithNativeModule:(rtc::scoped_refptr )module workerThread:(rtc::Thread *)workerThread; diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 5c116fae53..c88de392d7 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -77,7 +77,7 @@ - (instancetype)initWithNativeModule:(rtc::scoped_refptrBlockingCall([self] { NSArray *devices = [self _outputDevices]; @@ -92,11 +92,11 @@ - (RTCIODevice *)outputDevice { }); } -- (void)setOutputDevice: (RTCIODevice *)device { +- (void)setOutputDevice: (RTC_OBJC_TYPE(RTCIODevice) *)device { [self trySetOutputDevice: device]; } -- (BOOL)trySetOutputDevice: (RTCIODevice *)device { +- (BOOL)trySetOutputDevice: (RTC_OBJC_TYPE(RTCIODevice) *)device { return _workerThread->BlockingCall([self, device] { @@ -108,7 +108,7 @@ - (BOOL)trySetOutputDevice: (RTCIODevice *)device { } if (device != nil) { - index = [devices indexOfObjectPassingTest:^BOOL(RTCIODevice *e, NSUInteger i, BOOL *stop) { + index = [devices indexOfObjectPassingTest:^BOOL(RTC_OBJC_TYPE(RTCIODevice) *e, NSUInteger i, BOOL *stop) { return (*stop = [e.deviceId isEqualToString:device.deviceId]); }]; if (index == NSNotFound) { @@ -129,7 +129,7 @@ - (BOOL)trySetOutputDevice: (RTCIODevice *)device { }); } -- (RTCIODevice *)inputDevice { +- (RTC_OBJC_TYPE(RTCIODevice) *)inputDevice { return _workerThread->BlockingCall([self] { @@ -145,11 +145,11 @@ - (RTCIODevice *)inputDevice { }); } -- (void)setInputDevice: (RTCIODevice *)device { +- (void)setInputDevice: (RTC_OBJC_TYPE(RTCIODevice) *)device { [self trySetInputDevice: device]; } -- (BOOL)trySetInputDevice: (RTCIODevice *)device { +- (BOOL)trySetInputDevice: (RTC_OBJC_TYPE(RTCIODevice) *)device { return _workerThread->BlockingCall([self, device] { @@ -161,7 +161,7 @@ - (BOOL)trySetInputDevice: (RTCIODevice *)device { } if (device != nil) { - index = [devices indexOfObjectPassingTest:^BOOL(RTCIODevice *e, NSUInteger i, BOOL *stop) { + index = [devices indexOfObjectPassingTest:^BOOL(RTC_OBJC_TYPE(RTCIODevice) *e, NSUInteger i, BOOL *stop) { return (*stop = [e.deviceId isEqualToString:device.deviceId]); }]; if (index == NSNotFound) { @@ -261,7 +261,7 @@ - (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler _native->PlayoutDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCIODevice *device = [[RTCIODevice alloc] initWithType:RTCIODeviceTypeOutput deviceId:strGUID name:strName]; + RTC_OBJC_TYPE(RTCIODevice) *device = [[RTC_OBJC_TYPE(RTCIODevice) alloc] initWithType:RTCIODeviceTypeOutput deviceId:strGUID name:strName]; [result addObject: device]; } } @@ -283,7 +283,7 @@ - (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler _native->RecordingDeviceName(i, name, guid); NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; - RTCIODevice *device = [[RTCIODevice alloc] initWithType:RTCIODeviceTypeInput deviceId:strGUID name:strName]; + RTC_OBJC_TYPE(RTCIODevice) *device = [[RTC_OBJC_TYPE(RTCIODevice) alloc] initWithType:RTCIODeviceTypeInput deviceId:strGUID name:strName]; [result addObject: device]; } } diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 3d3af34893..d79972eae1 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -28,12 +28,12 @@ class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface { private: os_unfair_lock *lock_; - __weak RTCAudioTrack *audio_track_; + __weak RTC_OBJC_TYPE(RTCAudioTrack) *audio_track_; int64_t total_frames_ = 0; bool attached_ = false; public: - AudioSinkConverter(RTCAudioTrack *audioTrack, os_unfair_lock *lock) { + AudioSinkConverter(RTC_OBJC_TYPE(RTCAudioTrack) *audioTrack, os_unfair_lock *lock) { RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter init"; audio_track_ = audioTrack; lock_ = lock; @@ -151,6 +151,9 @@ void OnData(const void *audio_data, 0, NULL, &buffer); + // format is no longer required + CFRelease(format); + if (status != 0) { NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); return; @@ -209,7 +212,6 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { RTC_LOG(LS_INFO) << "RTCAudioTrack init"; - _lock = OS_UNFAIR_LOCK_INIT; _renderers = [NSHashTable weakObjectsHashTable]; _audioConverter = new rtc::RefCountedObject(self, &_lock); } @@ -271,7 +273,7 @@ - (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { NSArray *renderers = [_renderers allObjects]; os_unfair_lock_unlock(&_lock); - for (id renderer in renderers) { + for (id renderer in renderers) { [renderer renderSampleBuffer:sampleBuffer]; } } diff --git a/sdk/objc/api/peerconnection/RTCConfiguration.h b/sdk/objc/api/peerconnection/RTCConfiguration.h index 011eaa613d..b643f32f2a 100644 --- a/sdk/objc/api/peerconnection/RTCConfiguration.h +++ b/sdk/objc/api/peerconnection/RTCConfiguration.h @@ -255,6 +255,17 @@ RTC_OBJC_EXPORT */ @property(nonatomic, copy, nullable) NSNumber *iceInactiveTimeout; +/** + * When this flag is set, ports not bound to any specific network interface + * will be used, in addition to normal ports bound to the enumerated + * interfaces. Without this flag, these "any address" ports would only be + * used when network enumeration fails or is disabled. But under certain + * conditions, these ports may succeed where others fail, so they may allow + * the application to work in a wider variety of environments, at the expense + * of having to allocate additional candidates. + */ +@property(nonatomic, assign) BOOL enableIceGatheringOnAnyAddressPorts; + - (instancetype)init; @end diff --git a/sdk/objc/api/peerconnection/RTCConfiguration.mm b/sdk/objc/api/peerconnection/RTCConfiguration.mm index 86ecbabf8d..70bbc5d370 100644 --- a/sdk/objc/api/peerconnection/RTCConfiguration.mm +++ b/sdk/objc/api/peerconnection/RTCConfiguration.mm @@ -62,6 +62,7 @@ @implementation RTC_OBJC_TYPE (RTCConfiguration) @synthesize iceUnwritableTimeout = _iceUnwritableTimeout; @synthesize iceUnwritableMinChecks = _iceUnwritableMinChecks; @synthesize iceInactiveTimeout = _iceInactiveTimeout; +@synthesize enableIceGatheringOnAnyAddressPorts = _enableIceGatheringOnAnyAddressPorts; - (instancetype)init { // Copy defaults. @@ -156,6 +157,7 @@ - (instancetype)initWithNativeConfiguration: _iceInactiveTimeout = config.ice_inactive_timeout.has_value() ? [NSNumber numberWithInt:*config.ice_inactive_timeout] : nil; + _enableIceGatheringOnAnyAddressPorts = config.enable_any_address_ports; } return self; } @@ -303,6 +305,7 @@ - (NSString *)description { if (_iceInactiveTimeout != nil) { nativeConfig->ice_inactive_timeout = absl::optional(_iceInactiveTimeout.intValue); } + nativeConfig->enable_any_address_ports = _enableIceGatheringOnAnyAddressPorts; return nativeConfig.release(); } diff --git a/sdk/objc/api/peerconnection/RTCEncodedImage+Private.mm b/sdk/objc/api/peerconnection/RTCEncodedImage+Private.mm index c8936d3ad5..ac9946e55d 100644 --- a/sdk/objc/api/peerconnection/RTCEncodedImage+Private.mm +++ b/sdk/objc/api/peerconnection/RTCEncodedImage+Private.mm @@ -34,16 +34,16 @@ explicit ObjCEncodedImageBuffer(NSData *data) : data_(data) {} NSData *data_; }; -} +} // namespace // A simple wrapper around webrtc::EncodedImageBufferInterface to make it usable with associated // objects. -@interface RTCWrappedEncodedImageBuffer : NSObject +@interface RTC_OBJC_TYPE (RTCWrappedEncodedImageBuffer): NSObject @property(nonatomic) rtc::scoped_refptr buffer; - (instancetype)initWithEncodedImageBuffer: (rtc::scoped_refptr)buffer; @end -@implementation RTCWrappedEncodedImageBuffer +@implementation RTC_OBJC_TYPE (RTCWrappedEncodedImageBuffer) @synthesize buffer = _buffer; - (instancetype)initWithEncodedImageBuffer: (rtc::scoped_refptr)buffer { @@ -59,7 +59,7 @@ @implementation RTC_OBJC_TYPE (RTCEncodedImage) (Private) - (rtc::scoped_refptr)encodedData { - RTCWrappedEncodedImageBuffer *wrappedBuffer = + RTC_OBJC_TYPE(RTCWrappedEncodedImageBuffer) *wrappedBuffer = objc_getAssociatedObject(self, @selector(encodedData)); return wrappedBuffer.buffer; } @@ -68,7 +68,7 @@ - (void)setEncodedData:(rtc::scoped_refptr) return objc_setAssociatedObject( self, @selector(encodedData), - [[RTCWrappedEncodedImageBuffer alloc] initWithEncodedImageBuffer:buffer], + [[RTC_OBJC_TYPE(RTCWrappedEncodedImageBuffer) alloc] initWithEncodedImageBuffer:buffer], OBJC_ASSOCIATION_RETAIN_NONATOMIC); } diff --git a/sdk/objc/api/peerconnection/RTCFieldTrials.mm b/sdk/objc/api/peerconnection/RTCFieldTrials.mm index 193da9e4f7..b5a2eca8f0 100644 --- a/sdk/objc/api/peerconnection/RTCFieldTrials.mm +++ b/sdk/objc/api/peerconnection/RTCFieldTrials.mm @@ -10,47 +10,58 @@ #import "RTCFieldTrials.h" +#import #include - #import "base/RTCLogging.h" #include "system_wrappers/include/field_trial.h" NSString *const kRTCFieldTrialAudioForceABWENoTWCCKey = @"WebRTC-Audio-ABWENoTWCC"; -NSString * const kRTCFieldTrialFlexFec03AdvertisedKey = @"WebRTC-FlexFEC-03-Advertised"; -NSString * const kRTCFieldTrialFlexFec03Key = @"WebRTC-FlexFEC-03"; -NSString * const kRTCFieldTrialH264HighProfileKey = @"WebRTC-H264HighProfile"; -NSString * const kRTCFieldTrialMinimizeResamplingOnMobileKey = +NSString *const kRTCFieldTrialFlexFec03AdvertisedKey = @"WebRTC-FlexFEC-03-Advertised"; +NSString *const kRTCFieldTrialFlexFec03Key = @"WebRTC-FlexFEC-03"; +NSString *const kRTCFieldTrialH264HighProfileKey = @"WebRTC-H264HighProfile"; +NSString *const kRTCFieldTrialMinimizeResamplingOnMobileKey = @"WebRTC-Audio-MinimizeResamplingOnMobile"; NSString *const kRTCFieldTrialUseNWPathMonitor = @"WebRTC-Network-UseNWPathMonitor"; -NSString * const kRTCFieldTrialEnabledValue = @"Enabled"; +NSString *const kRTCFieldTrialEnabledValue = @"Enabled"; // InitFieldTrialsFromString stores the char*, so the char array must outlive // the application. static char *gFieldTrialInitString = nullptr; +static os_unfair_lock fieldTrialLock = OS_UNFAIR_LOCK_INIT; void RTCInitFieldTrialDictionary(NSDictionary *fieldTrials) { if (!fieldTrials) { RTCLogWarning(@"No fieldTrials provided."); return; } + // Assemble the keys and values into the field trial string. - // We don't perform any extra format checking. That should be done by the underlying WebRTC calls. NSMutableString *fieldTrialInitString = [NSMutableString string]; for (NSString *key in fieldTrials) { NSString *fieldTrialEntry = [NSString stringWithFormat:@"%@/%@/", key, fieldTrials[key]]; [fieldTrialInitString appendString:fieldTrialEntry]; } + size_t len = fieldTrialInitString.length + 1; + + // Locking before modifying global variable + os_unfair_lock_lock(&fieldTrialLock); if (gFieldTrialInitString != nullptr) { delete[] gFieldTrialInitString; + gFieldTrialInitString = nullptr; } + gFieldTrialInitString = new char[len]; - if (![fieldTrialInitString getCString:gFieldTrialInitString - maxLength:len - encoding:NSUTF8StringEncoding]) { + bool success = [fieldTrialInitString getCString:gFieldTrialInitString + maxLength:len + encoding:NSUTF8StringEncoding]; + if (!success) { RTCLogError(@"Failed to convert field trial string."); + os_unfair_lock_unlock(&fieldTrialLock); return; } + webrtc::field_trial::InitFieldTrialsFromString(gFieldTrialInitString); + os_unfair_lock_unlock(&fieldTrialLock); } diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptor+Private.h b/sdk/objc/api/peerconnection/RTCFrameCryptor+Private.h new file mode 100644 index 0000000000..86e6fdff8c --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCFrameCryptor+Private.h @@ -0,0 +1,45 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCFrameCryptor.h" + +#include +#include "api/crypto/frame_crypto_transformer.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCFrameCryptor) +() + + @end + +namespace webrtc { + +class RTCFrameCryptorDelegateAdapter : public FrameCryptorTransformerObserver { + public: + RTCFrameCryptorDelegateAdapter(RTC_OBJC_TYPE(RTCFrameCryptor) * frameCryptor); + ~RTCFrameCryptorDelegateAdapter() override; + + void OnFrameCryptionStateChanged(const std::string participant_id, + FrameCryptionState state) override; + + private: + __weak RTC_OBJC_TYPE(RTCFrameCryptor) * frame_cryptor_; +}; + +} // namespace webrtc + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptor.h b/sdk/objc/api/peerconnection/RTCFrameCryptor.h new file mode 100644 index 0000000000..864e55be95 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCFrameCryptor.h @@ -0,0 +1,78 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCRtpSender); +@class RTC_OBJC_TYPE(RTCRtpReceiver); +@class RTC_OBJC_TYPE(RTCFrameCryptorKeyProvider); +@class RTC_OBJC_TYPE(RTCFrameCryptor); +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); + +typedef NS_ENUM(NSUInteger, RTCCryptorAlgorithm) { + RTCCryptorAlgorithmAesGcm = 0, + RTCCryptorAlgorithmAesCbc, +}; + +typedef NS_ENUM(NSInteger, FrameCryptionState) { + FrameCryptionStateNew = 0, + FrameCryptionStateOk, + FrameCryptionStateEncryptionFailed, + FrameCryptionStateDecryptionFailed, + FrameCryptionStateMissingKey, + FrameCryptionStateKeyRatcheted, + FrameCryptionStateInternalError, +}; + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCFrameCryptorDelegate) + /** Called when the RTCFrameCryptor got errors. */ + - (void)frameCryptor + : (RTC_OBJC_TYPE(RTCFrameCryptor) *)frameCryptor didStateChangeWithParticipantId + : (NSString *)participantId withState : (FrameCryptionState)stateChanged; +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCFrameCryptor) : NSObject + +@property(nonatomic, assign) BOOL enabled; + +@property(nonatomic, assign) int keyIndex; + +@property(nonatomic, readonly) NSString *participantId; + +@property(nonatomic, weak, nullable) id delegate; + +- (nullable instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + rtpSender:(RTC_OBJC_TYPE(RTCRtpSender) *)sender + participantId:(NSString *)participantId + algorithm:(RTCCryptorAlgorithm)algorithm + keyProvider:(RTC_OBJC_TYPE(RTCFrameCryptorKeyProvider) *)keyProvider; + +- (nullable instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + rtpReceiver:(RTC_OBJC_TYPE(RTCRtpReceiver) *)receiver + participantId:(NSString *)participantId + algorithm:(RTCCryptorAlgorithm)algorithm + keyProvider:(RTC_OBJC_TYPE(RTCFrameCryptorKeyProvider) *)keyProvider; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptor.mm b/sdk/objc/api/peerconnection/RTCFrameCryptor.mm new file mode 100644 index 0000000000..c51b77e9c5 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCFrameCryptor.mm @@ -0,0 +1,224 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCFrameCryptor+Private.h" +#import "RTCFrameCryptorKeyProvider+Private.h" +#import "RTCPeerConnectionFactory+Private.h" +#import "RTCRtpReceiver+Private.h" +#import "RTCRtpSender+Private.h" + +#import +#include + +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +#include "api/crypto/frame_crypto_transformer.h" +#include "api/rtp_receiver_interface.h" +#include "api/rtp_sender_interface.h" + +namespace webrtc { + +RTCFrameCryptorDelegateAdapter::RTCFrameCryptorDelegateAdapter(RTC_OBJC_TYPE(RTCFrameCryptor) * + frameCryptor) + : frame_cryptor_(frameCryptor) {} + +RTCFrameCryptorDelegateAdapter::~RTCFrameCryptorDelegateAdapter() {} + +/* + kNew = 0, + kOk, + kEncryptionFailed, + kDecryptionFailed, + kMissingKey, + kInternalError, +*/ +void RTCFrameCryptorDelegateAdapter::OnFrameCryptionStateChanged(const std::string participant_id, + FrameCryptionState state) { + RTC_OBJC_TYPE(RTCFrameCryptor) *frameCryptor = frame_cryptor_; + if (frameCryptor.delegate) { + switch (state) { + case FrameCryptionState::kNew: + [frameCryptor.delegate frameCryptor:frameCryptor + didStateChangeWithParticipantId:[NSString stringForStdString:participant_id] + withState:FrameCryptionStateNew]; + break; + case FrameCryptionState::kOk: + [frameCryptor.delegate frameCryptor:frameCryptor + didStateChangeWithParticipantId:[NSString stringForStdString:participant_id] + withState:FrameCryptionStateOk]; + break; + case FrameCryptionState::kEncryptionFailed: + [frameCryptor.delegate frameCryptor:frameCryptor + didStateChangeWithParticipantId:[NSString stringForStdString:participant_id] + withState:FrameCryptionStateEncryptionFailed]; + break; + case FrameCryptionState::kDecryptionFailed: + [frameCryptor.delegate frameCryptor:frameCryptor + didStateChangeWithParticipantId:[NSString stringForStdString:participant_id] + withState:FrameCryptionStateDecryptionFailed]; + break; + case FrameCryptionState::kMissingKey: + [frameCryptor.delegate frameCryptor:frameCryptor + didStateChangeWithParticipantId:[NSString stringForStdString:participant_id] + withState:FrameCryptionStateMissingKey]; + break; + case FrameCryptionState::kKeyRatcheted: + [frameCryptor.delegate frameCryptor:frameCryptor + didStateChangeWithParticipantId:[NSString stringForStdString:participant_id] + withState:FrameCryptionStateKeyRatcheted]; + break; + case FrameCryptionState::kInternalError: + [frameCryptor.delegate frameCryptor:frameCryptor + didStateChangeWithParticipantId:[NSString stringForStdString:participant_id] + withState:FrameCryptionStateInternalError]; + break; + } + } +} +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCFrameCryptor) { + const webrtc::RtpSenderInterface *_sender; + const webrtc::RtpReceiverInterface *_receiver; + rtc::scoped_refptr _frame_crypto_transformer; + rtc::scoped_refptr _observer; + os_unfair_lock _lock; +} + +@synthesize participantId = _participantId; +@synthesize delegate = _delegate; + +- (webrtc::FrameCryptorTransformer::Algorithm)algorithmFromEnum:(RTCCryptorAlgorithm)algorithm { + switch (algorithm) { + case RTCCryptorAlgorithmAesGcm: + return webrtc::FrameCryptorTransformer::Algorithm::kAesGcm; + case RTCCryptorAlgorithmAesCbc: + return webrtc::FrameCryptorTransformer::Algorithm::kAesCbc; + default: + return webrtc::FrameCryptorTransformer::Algorithm::kAesGcm; + } +} + +- (nullable instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + rtpSender:(RTC_OBJC_TYPE(RTCRtpSender) *)sender + participantId:(NSString *)participantId + algorithm:(RTCCryptorAlgorithm)algorithm + keyProvider:(RTC_OBJC_TYPE(RTCFrameCryptorKeyProvider) *)keyProvider { + if (self = [super init]) { + _lock = OS_UNFAIR_LOCK_INIT; + + rtc::scoped_refptr nativeRtpSender = sender.nativeRtpSender; + if (nativeRtpSender == nullptr) return nil; + + rtc::scoped_refptr nativeTrack = nativeRtpSender->track(); + if (nativeTrack == nullptr) return nil; + + _observer = rtc::make_ref_counted(self); + _participantId = participantId; + + webrtc::FrameCryptorTransformer::MediaType mediaType = + nativeTrack->kind() == "audio" ? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame + : webrtc::FrameCryptorTransformer::MediaType::kVideoFrame; + + _frame_crypto_transformer = + rtc::scoped_refptr(new webrtc::FrameCryptorTransformer( + factory.signalingThread, [participantId stdString], mediaType, + [self algorithmFromEnum:algorithm], keyProvider.nativeKeyProvider)); + + nativeRtpSender->SetEncoderToPacketizerFrameTransformer(_frame_crypto_transformer); + _frame_crypto_transformer->SetEnabled(false); + _frame_crypto_transformer->RegisterFrameCryptorTransformerObserver(_observer); + } + + return self; +} + +- (nullable instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + rtpReceiver:(RTC_OBJC_TYPE(RTCRtpReceiver) *)receiver + participantId:(NSString *)participantId + algorithm:(RTCCryptorAlgorithm)algorithm + keyProvider:(RTC_OBJC_TYPE(RTCFrameCryptorKeyProvider) *)keyProvider { + if (self = [super init]) { + _lock = OS_UNFAIR_LOCK_INIT; + + rtc::scoped_refptr nativeRtpReceiver = receiver.nativeRtpReceiver; + if (nativeRtpReceiver == nullptr) return nil; + + rtc::scoped_refptr nativeTrack = nativeRtpReceiver->track(); + if (nativeTrack == nullptr) return nil; + + _observer = rtc::make_ref_counted(self); + _participantId = participantId; + + webrtc::FrameCryptorTransformer::MediaType mediaType = + nativeTrack->kind() == "audio" ? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame + : webrtc::FrameCryptorTransformer::MediaType::kVideoFrame; + + _frame_crypto_transformer = + rtc::scoped_refptr(new webrtc::FrameCryptorTransformer( + factory.signalingThread, [participantId stdString], mediaType, + [self algorithmFromEnum:algorithm], keyProvider.nativeKeyProvider)); + + nativeRtpReceiver->SetDepacketizerToDecoderFrameTransformer(_frame_crypto_transformer); + _frame_crypto_transformer->SetEnabled(false); + _frame_crypto_transformer->RegisterFrameCryptorTransformerObserver(_observer); + } + + return self; +} + +- (void)dealloc { + os_unfair_lock_lock(&_lock); + if (_frame_crypto_transformer != nullptr) { + _frame_crypto_transformer->UnRegisterFrameCryptorTransformerObserver(); + _frame_crypto_transformer = nullptr; + } + _observer = nullptr; + os_unfair_lock_unlock(&_lock); +} + +- (BOOL)enabled { + os_unfair_lock_lock(&_lock); + BOOL result = _frame_crypto_transformer != nullptr ? _frame_crypto_transformer->enabled() : NO; + os_unfair_lock_unlock(&_lock); + return result; +} + +- (void)setEnabled:(BOOL)enabled { + os_unfair_lock_lock(&_lock); + if (_frame_crypto_transformer != nullptr) { + _frame_crypto_transformer->SetEnabled(enabled); + } + os_unfair_lock_unlock(&_lock); +} + +- (int)keyIndex { + os_unfair_lock_lock(&_lock); + int result = _frame_crypto_transformer != nullptr ? _frame_crypto_transformer->key_index() : 0; + os_unfair_lock_unlock(&_lock); + return result; +} + +- (void)setKeyIndex:(int)keyIndex { + os_unfair_lock_lock(&_lock); + if (_frame_crypto_transformer != nullptr) { + _frame_crypto_transformer->SetKeyIndex(keyIndex); + } + os_unfair_lock_unlock(&_lock); +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider+Private.h b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider+Private.h new file mode 100644 index 0000000000..eb7c83e2e7 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider+Private.h @@ -0,0 +1,31 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCFrameCryptorKeyProvider.h" + +#include "api/crypto/frame_crypto_transformer.h" +#include "rtc_base/ref_count.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCFrameCryptorKeyProvider) +() + + @property(nonatomic, readonly) rtc::scoped_refptr nativeKeyProvider; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h new file mode 100644 index 0000000000..6443b23349 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h @@ -0,0 +1,62 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCFrameCryptorKeyProvider) : NSObject + +- (void)setSharedKey:(NSData *)key withIndex:(int)index; + +- (NSData *)ratchetSharedKey:(int)index; + +- (NSData *)exportSharedKey:(int)index; + +- (void)setKey:(NSData *)key withIndex:(int)index forParticipant:(NSString *)participantId; + +- (NSData *)ratchetKey:(NSString *)participantId withIndex:(int)index; + +- (NSData *)exportKey:(NSString *)participantId withIndex:(int)index; + +- (void)setSifTrailer:(NSData *)trailer; + +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes; + +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes + failureTolerance:(int)failureTolerance + keyRingSize:(int)keyRingSize; + +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes + failureTolerance:(int)failureTolerance + keyRingSize:(int)keyRingSize + discardFrameWhenCryptorNotReady:(BOOL)discardFrameWhenCryptorNotReady; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm new file mode 100644 index 0000000000..88bebfcd9d --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm @@ -0,0 +1,124 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCFrameCryptorKeyProvider+Private.h" + +#include +#include "api/crypto/frame_crypto_transformer.h" + +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCFrameCryptorKeyProvider) { + rtc::scoped_refptr _nativeKeyProvider; +} + +- (rtc::scoped_refptr)nativeKeyProvider { + return _nativeKeyProvider; +} + +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(NSData *)uncryptedMagicBytes { + return [self initWithRatchetSalt:salt + ratchetWindowSize:windowSize + sharedKeyMode:sharedKey + uncryptedMagicBytes:uncryptedMagicBytes + failureTolerance:-1 + keyRingSize:webrtc::DEFAULT_KEYRING_SIZE]; +} + +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes + failureTolerance:(int)failureTolerance + keyRingSize:(int)keyRingSize { + return [self initWithRatchetSalt:salt + ratchetWindowSize:windowSize + sharedKeyMode:sharedKey + uncryptedMagicBytes:uncryptedMagicBytes + failureTolerance:-1 + keyRingSize:keyRingSize + discardFrameWhenCryptorNotReady:false]; +} + +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes + failureTolerance:(int)failureTolerance + keyRingSize:(int)keyRingSize + discardFrameWhenCryptorNotReady:(BOOL)discardFrameWhenCryptorNotReady { + if (self = [super init]) { + webrtc::KeyProviderOptions options; + options.ratchet_salt = std::vector((const uint8_t *)salt.bytes, + ((const uint8_t *)salt.bytes) + salt.length); + options.ratchet_window_size = windowSize; + options.shared_key = sharedKey; + options.failure_tolerance = failureTolerance; + options.key_ring_size = keyRingSize; + options.discard_frame_when_cryptor_not_ready = discardFrameWhenCryptorNotReady; + if(uncryptedMagicBytes != nil) { + options.uncrypted_magic_bytes = std::vector((const uint8_t *)uncryptedMagicBytes.bytes, + ((const uint8_t *)uncryptedMagicBytes.bytes) + uncryptedMagicBytes.length); + } + _nativeKeyProvider = rtc::make_ref_counted(options); + } + return self; +} + +- (void)setKey:(NSData *)key withIndex:(int)index forParticipant:(NSString *)participantId { + _nativeKeyProvider->SetKey( + [participantId stdString], + index, + std::vector((const uint8_t *)key.bytes, ((const uint8_t *)key.bytes) + key.length)); +} + +- (void)setSharedKey:(NSData *)key withIndex:(int)index { + _nativeKeyProvider->SetSharedKey( + index, + std::vector((const uint8_t *)key.bytes, ((const uint8_t *)key.bytes) + key.length)); +} + +- (NSData *)ratchetSharedKey:(int)index { + std::vector nativeKey = _nativeKeyProvider->RatchetSharedKey(index); + return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; +} + +- (NSData *)exportSharedKey:(int)index { + std::vector nativeKey = _nativeKeyProvider->ExportSharedKey(index); + return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; +} + +- (NSData *)ratchetKey:(NSString *)participantId withIndex:(int)index { + std::vector nativeKey = _nativeKeyProvider->RatchetKey([participantId stdString], index); + return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; +} + +- (NSData *)exportKey:(NSString *)participantId withIndex:(int)index { + std::vector nativeKey = _nativeKeyProvider->ExportKey([participantId stdString], index); + return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; +} + +- (void)setSifTrailer:(NSData *)trailer { + _nativeKeyProvider->SetSifTrailer( + std::vector((const uint8_t *)trailer.bytes, + ((const uint8_t *)trailer.bytes) + trailer.length)); +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCIODevice+Private.h b/sdk/objc/api/peerconnection/RTCIODevice+Private.h index 0eb09b83a4..e736c993e1 100644 --- a/sdk/objc/api/peerconnection/RTCIODevice+Private.h +++ b/sdk/objc/api/peerconnection/RTCIODevice+Private.h @@ -17,7 +17,7 @@ NS_ASSUME_NONNULL_BEGIN -@interface RTCIODevice () +@interface RTC_OBJC_TYPE(RTCIODevice) () - (instancetype)initWithType:(RTCIODeviceType)type deviceId:(NSString *)deviceId diff --git a/sdk/objc/api/peerconnection/RTCIODevice.mm b/sdk/objc/api/peerconnection/RTCIODevice.mm index 27e1255e8e..b3738f71fe 100644 --- a/sdk/objc/api/peerconnection/RTCIODevice.mm +++ b/sdk/objc/api/peerconnection/RTCIODevice.mm @@ -19,7 +19,7 @@ NSString *const kDefaultDeviceId = @"default"; -@implementation RTCIODevice +@implementation RTC_OBJC_TYPE(RTCIODevice) @synthesize type = _type; @synthesize deviceId = _deviceId; diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h index 8194cfe4ba..53c5a1a12c 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h @@ -25,6 +25,10 @@ NS_ASSUME_NONNULL_BEGIN @class RTC_OBJC_TYPE(RTCVideoTrack); @class RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions); @class RTC_OBJC_TYPE(RTCAudioDeviceModule); +@class RTC_OBJC_TYPE(RTCRtpCapabilities); + +typedef NS_ENUM(NSInteger, RTCRtpMediaType); + @protocol RTC_OBJC_TYPE (RTCPeerConnectionDelegate); @protocol RTC_OBJC_TYPE @@ -63,6 +67,14 @@ RTC_OBJC_EXPORT audioProcessingModule: (nullable id)audioProcessingModule; +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioDeviceModule) *audioDeviceModule; + +/** + * Valid kind values are kRTCMediaStreamTrackKindAudio and + * kRTCMediaStreamTrackKindVideo. + */ +- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpSenderCapabilitiesForKind:(NSString *)kind; + @property(nonatomic, readonly) RTCAudioDeviceModule *audioDeviceModule; /** Initialize an RTCAudioSource with constraints. */ diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index 15bfa9fa0b..08200f46fd 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -25,6 +25,9 @@ #import "RTCPeerConnection+Private.h" #import "RTCVideoSource+Private.h" #import "RTCVideoTrack+Private.h" +#import "RTCRtpReceiver+Private.h" +#import "RTCRtpCapabilities+Private.h" +#import "RTCRtpCodecCapability+Private.h" #import "base/RTCLogging.h" #import "base/RTCVideoDecoderFactory.h" #import "base/RTCVideoEncoderFactory.h" @@ -64,7 +67,7 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { std::unique_ptr _workerThread; std::unique_ptr _signalingThread; rtc::scoped_refptr _nativeAudioDeviceModule; - RTCDefaultAudioProcessingModule *_defaultAudioProcessingModule; + RTC_OBJC_TYPE(RTCDefaultAudioProcessingModule) *_defaultAudioProcessingModule; BOOL _hasStartedAecDump; } @@ -130,12 +133,26 @@ - (instancetype)init { #endif } +- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpSenderCapabilitiesFor:(RTCRtpMediaType)mediaType { + + webrtc::RtpCapabilities capabilities = _nativeFactory->GetRtpSenderCapabilities([RTC_OBJC_TYPE(RTCRtpReceiver) nativeMediaTypeForMediaType: mediaType]); + + return [[RTCRtpCapabilities alloc] initWithNativeRtpCapabilities: capabilities]; +} + +- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpReceiverCapabilitiesFor:(RTCRtpMediaType)mediaType { + + webrtc::RtpCapabilities capabilities = _nativeFactory->GetRtpReceiverCapabilities([RTC_OBJC_TYPE(RTCRtpReceiver) nativeMediaTypeForMediaType: mediaType]); + + return [[RTCRtpCapabilities alloc] initWithNativeRtpCapabilities: capabilities]; +} + - (instancetype) initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing encoderFactory:(nullable id)encoderFactory decoderFactory:(nullable id)decoderFactory audioProcessingModule: - (nullable id)audioProcessingModule{ + (nullable id)audioProcessingModule { #ifdef HAVE_NO_MEDIA return [self initWithNoMedia]; #else @@ -148,12 +165,21 @@ - (instancetype)init { native_decoder_factory = webrtc::ObjCToNativeVideoDecoderFactory(decoderFactory); } rtc::scoped_refptr audio_device_module = [self createAudioDeviceModule:bypassVoiceProcessing]; + + if ([audioProcessingModule isKindOfClass:[RTC_OBJC_TYPE(RTCDefaultAudioProcessingModule) class]]) { + _defaultAudioProcessingModule = (RTC_OBJC_TYPE(RTCDefaultAudioProcessingModule) *)audioProcessingModule; + } else { + _defaultAudioProcessingModule = [[RTC_OBJC_TYPE(RTCDefaultAudioProcessingModule) alloc] init]; + } + + NSLog(@"AudioProcessingModule: %@", _defaultAudioProcessingModule); + return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory() nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() nativeVideoEncoderFactory:std::move(native_encoder_factory) nativeVideoDecoderFactory:std::move(native_decoder_factory) audioDeviceModule:audio_device_module.get() - audioProcessingModule:nullptr + audioProcessingModule:_defaultAudioProcessingModule.nativeAudioProcessingModule bypassVoiceProcessing:bypassVoiceProcessing]; #endif } @@ -240,7 +266,7 @@ - (instancetype)initWithNativeAudioEncoderFactory: dependencies.trials = std::make_unique(); dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(dependencies.trials.get()); - + // always create ADM on worker thread _nativeAudioDeviceModule = _workerThread->BlockingCall([&dependencies, &bypassVoiceProcessing]() { return webrtc::AudioDeviceModule::Create(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, @@ -248,16 +274,14 @@ - (instancetype)initWithNativeAudioEncoderFactory: bypassVoiceProcessing == YES); }); - _audioDeviceModule = [[RTCAudioDeviceModule alloc] initWithNativeModule: _nativeAudioDeviceModule + _audioDeviceModule = [[RTC_OBJC_TYPE(RTCAudioDeviceModule) alloc] initWithNativeModule: _nativeAudioDeviceModule workerThread: _workerThread.get()]; + dependencies.adm = _nativeAudioDeviceModule; + dependencies.audio_encoder_factory = std::move(audioEncoderFactory); + dependencies.audio_decoder_factory = std::move(audioDecoderFactory); + dependencies.video_encoder_factory = std::move(videoEncoderFactory); + dependencies.video_decoder_factory = std::move(videoDecoderFactory); - media_deps.adm = _nativeAudioDeviceModule; - media_deps.task_queue_factory = dependencies.task_queue_factory.get(); - media_deps.audio_encoder_factory = std::move(audioEncoderFactory); - media_deps.audio_decoder_factory = std::move(audioDecoderFactory); - media_deps.video_encoder_factory = std::move(videoEncoderFactory); - media_deps.video_decoder_factory = std::move(videoDecoderFactory); - if (audioProcessingModule) { dependencies.audio_processing = std::move(audioProcessingModule); } else { diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h index 070a0e74a5..4d7025bf93 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h @@ -12,9 +12,9 @@ NS_ASSUME_NONNULL_BEGIN -@interface RTCPeerConnectionFactoryBuilder (DefaultComponents) +@interface RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) (DefaultComponents) -+ (RTCPeerConnectionFactoryBuilder *)defaultBuilder; ++ (RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) *)defaultBuilder; @end diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm index 522e520e12..a2f633e1a4 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm @@ -22,10 +22,10 @@ #import "sdk/objc/native/api/audio_device_module.h" #endif -@implementation RTCPeerConnectionFactoryBuilder (DefaultComponents) +@implementation RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) (DefaultComponents) -+ (RTCPeerConnectionFactoryBuilder *)defaultBuilder { - RTCPeerConnectionFactoryBuilder *builder = [[RTCPeerConnectionFactoryBuilder alloc] init]; ++ (RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) *)defaultBuilder { + RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) *builder = [[RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) alloc] init]; auto audioEncoderFactory = webrtc::CreateBuiltinAudioEncoderFactory(); [builder setAudioEncoderFactory:audioEncoderFactory]; diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h index f0b0de156a..a46839b6b3 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h @@ -25,9 +25,9 @@ class AudioProcessing; NS_ASSUME_NONNULL_BEGIN -@interface RTCPeerConnectionFactoryBuilder : NSObject +@interface RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) : NSObject -+ (RTCPeerConnectionFactoryBuilder *)builder; ++ (RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) *)builder; - (RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)createPeerConnectionFactory; diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm index 0981fb3879..4cb12b0a59 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm @@ -18,7 +18,7 @@ #include "modules/audio_device/include/audio_device.h" #include "modules/audio_processing/include/audio_processing.h" -@implementation RTCPeerConnectionFactoryBuilder { +@implementation RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) { std::unique_ptr _videoEncoderFactory; std::unique_ptr _videoDecoderFactory; rtc::scoped_refptr _audioEncoderFactory; @@ -27,8 +27,8 @@ @implementation RTCPeerConnectionFactoryBuilder { rtc::scoped_refptr _audioProcessingModule; } -+ (RTCPeerConnectionFactoryBuilder *)builder { - return [[RTCPeerConnectionFactoryBuilder alloc] init]; ++ (RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) *)builder { + return [[RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) alloc] init]; } - (RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)createPeerConnectionFactory { diff --git a/sdk/objc/api/peerconnection/RTCRtpCodecParameters.h b/sdk/objc/api/peerconnection/RTCRtpCodecParameters.h index 6135223720..4d24d3ccd6 100644 --- a/sdk/objc/api/peerconnection/RTCRtpCodecParameters.h +++ b/sdk/objc/api/peerconnection/RTCRtpCodecParameters.h @@ -30,6 +30,7 @@ RTC_EXTERN const NSString *const kRTCComfortNoiseCodecName; RTC_EXTERN const NSString *const kRTCVp8CodecName; RTC_EXTERN const NSString *const kRTCVp9CodecName; RTC_EXTERN const NSString *const kRTCH264CodecName; +RTC_EXTERN const NSString *const kRTCAv1CodecName; /** Defined in https://www.w3.org/TR/webrtc/#idl-def-rtcrtpcodecparameters */ RTC_OBJC_EXPORT diff --git a/sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm b/sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm index 6201e57b93..42a310cb79 100644 --- a/sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm +++ b/sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm @@ -32,6 +32,7 @@ const NSString * const kRTCVp8CodecName = @(cricket::kVp8CodecName); const NSString * const kRTCVp9CodecName = @(cricket::kVp9CodecName); const NSString * const kRTCH264CodecName = @(cricket::kH264CodecName); +const NSString * const kRTCAv1CodecName = @(cricket::kAv1CodecName); @implementation RTC_OBJC_TYPE (RTCRtpCodecParameters) diff --git a/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h b/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h index 07f6b7a39c..af0c6993bc 100644 --- a/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h +++ b/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h @@ -69,6 +69,10 @@ RTC_OBJC_EXPORT https://w3c.github.io/webrtc-extensions/#dom-rtcrtpencodingparameters-adaptiveptime */ @property(nonatomic, assign) BOOL adaptiveAudioPacketTime; +/** A case-sensitive identifier of the scalability mode to be used for this stream. + https://w3c.github.io/webrtc-svc/#rtcrtpencodingparameters */ +@property(nonatomic, copy, nullable) NSString *scalabilityMode; + - (instancetype)init; @end diff --git a/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm b/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm index d6087dafb0..aecb88b6f6 100644 --- a/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm +++ b/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm @@ -25,6 +25,7 @@ @implementation RTC_OBJC_TYPE (RTCRtpEncodingParameters) @synthesize bitratePriority = _bitratePriority; @synthesize networkPriority = _networkPriority; @synthesize adaptiveAudioPacketTime = _adaptiveAudioPacketTime; +@synthesize scalabilityMode = _scalabilityMode; - (instancetype)init { webrtc::RtpEncodingParameters nativeParameters; @@ -59,6 +60,9 @@ - (instancetype)initWithNativeParameters: if (nativeParameters.ssrc) { _ssrc = [NSNumber numberWithUnsignedLong:*nativeParameters.ssrc]; } + if (nativeParameters.scalability_mode) { + _scalabilityMode = [NSString stringWithUTF8String:nativeParameters.scalability_mode->c_str()]; + } _bitratePriority = nativeParameters.bitrate_priority; _networkPriority = [RTC_OBJC_TYPE(RTCRtpEncodingParameters) priorityFromNativePriority:nativeParameters.network_priority]; @@ -92,6 +96,9 @@ - (instancetype)initWithNativeParameters: if (_ssrc != nil) { parameters.ssrc = absl::optional(_ssrc.unsignedLongValue); } + if (_scalabilityMode != nil) { + parameters.scalability_mode = absl::optional(std::string([_scalabilityMode UTF8String])); + } parameters.bitrate_priority = _bitratePriority; parameters.network_priority = [RTC_OBJC_TYPE(RTCRtpEncodingParameters) nativePriorityFromPriority:_networkPriority]; diff --git a/sdk/objc/api/peerconnection/RTCRtpTransceiver.h b/sdk/objc/api/peerconnection/RTCRtpTransceiver.h index 1701f680a4..fca088be7e 100644 --- a/sdk/objc/api/peerconnection/RTCRtpTransceiver.h +++ b/sdk/objc/api/peerconnection/RTCRtpTransceiver.h @@ -14,6 +14,8 @@ #import "RTCRtpReceiver.h" #import "RTCRtpSender.h" +@class RTC_OBJC_TYPE(RTCRtpCodecCapability); + NS_ASSUME_NONNULL_BEGIN extern NSString *const kRTCRtpTransceiverErrorDomain; @@ -105,6 +107,8 @@ RTC_OBJC_EXPORT */ @property(nonatomic, readonly) RTCRtpTransceiverDirection direction; +@property(nonatomic, copy) NSArray *codecPreferences; + /** The currentDirection attribute indicates the current direction negotiated * for this transceiver. If this transceiver has never been represented in an * offer/answer exchange, or if the transceiver is stopped, the value is not diff --git a/sdk/objc/api/peerconnection/RTCRtpTransceiver.mm b/sdk/objc/api/peerconnection/RTCRtpTransceiver.mm index eb0ad96738..dc51149934 100644 --- a/sdk/objc/api/peerconnection/RTCRtpTransceiver.mm +++ b/sdk/objc/api/peerconnection/RTCRtpTransceiver.mm @@ -15,6 +15,8 @@ #import "RTCRtpParameters+Private.h" #import "RTCRtpReceiver+Private.h" #import "RTCRtpSender+Private.h" +#import "RTCRtpCodecCapability.h" +#import "RTCRtpCodecCapability+Private.h" #import "base/RTCLogging.h" #import "helpers/NSString+StdString.h" @@ -68,6 +70,20 @@ - (NSString *)mid { } } +- (NSArray *)codecPreferences { + + NSMutableArray *result = [NSMutableArray array]; + + std::vector capabilities = _nativeRtpTransceiver->codec_preferences(); + + for (auto & element : capabilities) { + RTC_OBJC_TYPE(RTCRtpCodecCapability) *object = [[RTC_OBJC_TYPE(RTCRtpCodecCapability) alloc] initWithNativeRtpCodecCapability: element]; + [result addObject: object]; + } + + return result; +} + @synthesize sender = _sender; @synthesize receiver = _receiver; diff --git a/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm b/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm index 2eb8d366d2..88c2add1df 100644 --- a/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm +++ b/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm @@ -12,6 +12,11 @@ #import "helpers/NSString+StdString.h" +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "modules/video_coding/svc/create_scalability_structure.h" + @implementation RTC_OBJC_TYPE (RTCVideoCodecInfo) (Private) @@ -31,8 +36,16 @@ - (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format { std::string value = [NSString stdStringForString:self.parameters[paramKey]]; parameters[key] = value; } - - return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters); + + absl::InlinedVector + scalability_modes; + for (NSString *scalabilityMode in self.scalabilityModes) { + auto scalability_mode = webrtc::ScalabilityModeFromString([NSString stdStringForString:scalabilityMode]); + if (scalability_mode != absl::nullopt) { + scalability_modes.push_back(*scalability_mode); + } + } + return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters, scalability_modes); } @end diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index df294d2f3e..546ec80a61 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -53,7 +53,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto } - (void)dealloc { - for (RTCVideoRendererAdapter *adapter in _adapters) { + for (RTC_OBJC_TYPE(RTCVideoRendererAdapter) * adapter in _adapters) { self.nativeVideoTrack->RemoveSink(adapter.nativeVideoRenderer); } } @@ -85,18 +85,17 @@ - (void)addRenderer:(id)renderer { } // Make sure we don't have this renderer yet. - for (RTCVideoRendererAdapter *adapter in _adapters) { + for (RTC_OBJC_TYPE(RTCVideoRendererAdapter) * adapter in _adapters) { if (adapter.videoRenderer == renderer) { RTC_LOG(LS_INFO) << "|renderer| is already attached to this track"; return; } } // Create a wrapper that provides a native pointer for us. - RTCVideoRendererAdapter* adapter = - [[RTCVideoRendererAdapter alloc] initWithNativeRenderer:renderer]; + RTC_OBJC_TYPE(RTCVideoRendererAdapter) *adapter = + [[RTC_OBJC_TYPE(RTCVideoRendererAdapter) alloc] initWithNativeRenderer:renderer]; [_adapters addObject:adapter]; - self.nativeVideoTrack->AddOrUpdateSink(adapter.nativeVideoRenderer, - rtc::VideoSinkWants()); + self.nativeVideoTrack->AddOrUpdateSink(adapter.nativeVideoRenderer, rtc::VideoSinkWants()); } - (void)removeRenderer:(id)renderer { @@ -105,9 +104,8 @@ - (void)removeRenderer:(id)renderer { return; } __block NSUInteger indexToRemove = NSNotFound; - [_adapters enumerateObjectsUsingBlock:^(RTCVideoRendererAdapter *adapter, - NSUInteger idx, - BOOL *stop) { + [_adapters enumerateObjectsUsingBlock:^( + RTC_OBJC_TYPE(RTCVideoRendererAdapter) * adapter, NSUInteger idx, BOOL * stop) { if (adapter.videoRenderer == renderer) { indexToRemove = idx; *stop = YES; @@ -117,8 +115,7 @@ - (void)removeRenderer:(id)renderer { RTC_LOG(LS_INFO) << "removeRenderer called with a renderer that has not been previously added"; return; } - RTCVideoRendererAdapter *adapterToRemove = - [_adapters objectAtIndex:indexToRemove]; + RTC_OBJC_TYPE(RTCVideoRendererAdapter) *adapterToRemove = [_adapters objectAtIndex:indexToRemove]; self.nativeVideoTrack->RemoveSink(adapterToRemove.nativeVideoRenderer); [_adapters removeObjectAtIndex:indexToRemove]; } diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h index 8aa55e4bfa..a900728049 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h +++ b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h @@ -24,4 +24,6 @@ RTC_OBJC_EXPORT + (bool)isSupported; ++ (NSArray *)scalabilityModes; + @end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm index 43137b5bc8..662a3ca694 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm +++ b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm @@ -15,7 +15,11 @@ #import "RTCNativeVideoEncoder.h" #import "RTCNativeVideoEncoderBuilder+Native.h" #import "RTCVideoEncoderAV1.h" +#import "helpers/NSString+StdString.h" +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" +#include "modules/video_coding/svc/create_scalability_structure.h" @interface RTC_OBJC_TYPE (RTCVideoEncoderAV1Builder) : RTC_OBJC_TYPE(RTCNativeVideoEncoder) @@ -39,4 +43,14 @@ + (bool)isSupported { return true; } + + (NSArray *)scalabilityModes { + NSMutableArray *scalabilityModes = [NSMutableArray array]; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + [scalabilityModes addObject:[NSString stringForAbslStringView:webrtc::ScalabilityModeToString(scalability_mode)]]; + } + } + return scalabilityModes; + } + @end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h index c3e3d4538f..4f1b55c713 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h +++ b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h @@ -10,4 +10,4 @@ RTC_OBJC_EXPORT fallback:(id)fallback videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo; -@end \ No newline at end of file +@end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm index 568e1bd517..8ca988562c 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm +++ b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm @@ -2,25 +2,63 @@ #import "RTCMacros.h" #import "RTCVideoEncoderSimulcast.h" -#import "RTCWrappedNativeVideoEncoder.h" +#import "RTCNativeVideoEncoder.h" +#import "RTCNativeVideoEncoderBuilder+Native.h" #import "api/peerconnection/RTCVideoCodecInfo+Private.h" +#include "api/transport/field_trial_based_config.h" #include "native/api/video_encoder_factory.h" #include "media/engine/simulcast_encoder_adapter.h" +@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcastBuilder) + : RTC_OBJC_TYPE(RTCNativeVideoEncoder) { + + id _primary; + id _fallback; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *_videoCodecInfo; +} + +- (id)initWithPrimary:(id)primary + fallback:(id)fallback + videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo; + +@end + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcastBuilder) + +- (std::unique_ptr)build:(const webrtc::Environment&)env { + auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(_primary); + auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(_fallback); + auto nativeFormat = [_videoCodecInfo nativeSdpVideoFormat]; + return std::make_unique( + env, + nativePrimary.release(), + nativeFallback.release(), + std::move(nativeFormat)); +} + +- (id)initWithPrimary:(id)primary + fallback:(id)fallback + videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo { + if (self = [super init]) { + self->_primary = primary; + self->_fallback = fallback; + self->_videoCodecInfo = videoCodecInfo; + } + return self; +} + +@end + @implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) + (id)simulcastEncoderWithPrimary:(id)primary fallback:(id)fallback videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo { - auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(primary); - auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(fallback); - auto nativeFormat = [videoCodecInfo nativeSdpVideoFormat]; - return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] - initWithNativeEncoder: std::make_unique( - nativePrimary.release(), - nativeFallback.release(), - std::move(nativeFormat))]; + return [[RTC_OBJC_TYPE(RTCVideoEncoderSimulcastBuilder) alloc] + initWithPrimary:primary + fallback:fallback + videoCodecInfo:videoCodecInfo]; } -@end \ No newline at end of file +@end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h index f7dac6117d..adfca0f9a4 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h +++ b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h @@ -24,4 +24,6 @@ RTC_OBJC_EXPORT + (bool)isSupported; ++ (NSArray *)scalabilityModes; + @end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm index 1026f40092..3055017024 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm +++ b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm @@ -16,7 +16,12 @@ #import "RTCNativeVideoEncoderBuilder+Native.h" #import "RTCVideoEncoderVP9.h" +#import "helpers/NSString+StdString.h" + +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "modules/video_coding/svc/create_scalability_structure.h" @interface RTC_OBJC_TYPE (RTCVideoEncoderVP9Builder) : RTC_OBJC_TYPE(RTCNativeVideoEncoder) @@ -47,5 +52,13 @@ + (bool)isSupported { return false; #endif } - + + (NSArray *)scalabilityModes { + NSMutableArray *scalabilityModes = [NSMutableArray array]; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + [scalabilityModes addObject:[NSString stringForAbslStringView:webrtc::ScalabilityModeToString(scalability_mode)]]; + } + } + return scalabilityModes; + } @end diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h index def20eac3c..3669831fca 100644 --- a/sdk/objc/base/RTCAudioRenderer.h +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -23,11 +23,9 @@ NS_ASSUME_NONNULL_BEGIN -RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE -(RTCAudioRenderer) +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE(RTCAudioRenderer) - - (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer - NS_SWIFT_NAME(render(sampleBuffer:)); +- (void)renderSampleBuffer: (CMSampleBufferRef)sampleBuffer NS_SWIFT_NAME(render(sampleBuffer:)); @end diff --git a/sdk/objc/base/RTCVideoCodecInfo.h b/sdk/objc/base/RTCVideoCodecInfo.h index fa28958f25..9da0c7aa81 100644 --- a/sdk/objc/base/RTCVideoCodecInfo.h +++ b/sdk/objc/base/RTCVideoCodecInfo.h @@ -26,10 +26,16 @@ RTC_OBJC_EXPORT parameters:(nullable NSDictionary *)parameters NS_DESIGNATED_INITIALIZER; +- (instancetype)initWithName:(NSString *)name + parameters:(nullable NSDictionary *)parameters + scalabilityModes:(nullable NSArray *)scalabilityModes + NS_DESIGNATED_INITIALIZER; + - (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info; @property(nonatomic, readonly) NSString *name; @property(nonatomic, readonly) NSDictionary *parameters; +@property(nonatomic, readonly) NSArray *scalabilityModes; @end diff --git a/sdk/objc/base/RTCVideoCodecInfo.m b/sdk/objc/base/RTCVideoCodecInfo.m index ce26ae1de3..441ecde7c5 100644 --- a/sdk/objc/base/RTCVideoCodecInfo.m +++ b/sdk/objc/base/RTCVideoCodecInfo.m @@ -14,6 +14,7 @@ @implementation RTC_OBJC_TYPE (RTCVideoCodecInfo) @synthesize name = _name; @synthesize parameters = _parameters; +@synthesize scalabilityModes = _scalabilityModes; - (instancetype)initWithName:(NSString *)name { return [self initWithName:name parameters:nil]; @@ -24,15 +25,29 @@ - (instancetype)initWithName:(NSString *)name if (self = [super init]) { _name = name; _parameters = (parameters ? parameters : @{}); + _scalabilityModes = @[]; } return self; } +- (instancetype)initWithName:(NSString *)name + parameters:(nullable NSDictionary *)parameters + scalabilityModes:(nullable NSArray *)scalabilityModes { + if (self = [super init]) { + _name = name; + _parameters = (parameters ? parameters : @{}); + _scalabilityModes = (scalabilityModes ? scalabilityModes : @[]); + } + + return self; + } + - (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { if (!info || ![self.name isEqualToString:info.name] || - ![self.parameters isEqualToDictionary:info.parameters]) { + ![self.parameters isEqualToDictionary:info.parameters] || + ![self.scalabilityModes isEqualToArray:info.scalabilityModes]) { return NO; } return YES; diff --git a/sdk/objc/components/audio/RTCAudioSession+Configuration.mm b/sdk/objc/components/audio/RTCAudioSession+Configuration.mm index 4bf0a542e1..f7b45b5850 100644 --- a/sdk/objc/components/audio/RTCAudioSession+Configuration.mm +++ b/sdk/objc/components/audio/RTCAudioSession+Configuration.mm @@ -63,6 +63,33 @@ - (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configur } } + if (self.mode != configuration.mode) { + NSError *modeError = nil; + if (![self setMode:configuration.mode error:&modeError]) { + RTCLogError(@"Failed to set mode to %@: %@", + self.mode, + modeError.localizedDescription); + error = modeError; + } else { + RTCLog(@"Set mode to: %@", configuration.mode); + } + } + + // Sometimes category options don't stick after setting mode. + if (self.categoryOptions != configuration.categoryOptions) { + NSError *categoryError = nil; + if (![self setCategory:configuration.category + withOptions:configuration.categoryOptions + error:&categoryError]) { + RTCLogError(@"Failed to set category options: %@", + categoryError.localizedDescription); + error = categoryError; + } else { + RTCLog(@"Set category options to: %ld", + (long)configuration.categoryOptions); + } + } + if (self.preferredSampleRate != configuration.sampleRate) { NSError *sampleRateError = nil; if (![self setPreferredSampleRate:configuration.sampleRate diff --git a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h index 6a75f01479..1799c11415 100644 --- a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h +++ b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h @@ -19,7 +19,7 @@ class AudioSessionObserver; /** Adapter that forwards RTCAudioSessionDelegate calls to the appropriate * methods on the AudioSessionObserver. */ -@interface RTCNativeAudioSessionDelegateAdapter : NSObject +@interface RTC_OBJC_TYPE(RTCNativeAudioSessionDelegateAdapter) : NSObject - (instancetype)init NS_UNAVAILABLE; diff --git a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm index daddf314a4..f652ad1e5f 100644 --- a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm +++ b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm @@ -14,7 +14,7 @@ #import "base/RTCLogging.h" -@implementation RTCNativeAudioSessionDelegateAdapter { +@implementation RTC_OBJC_TYPE(RTCNativeAudioSessionDelegateAdapter) { webrtc::AudioSessionObserver *_observer; } diff --git a/sdk/objc/components/capturer/RTCCameraVideoCapturer.h b/sdk/objc/components/capturer/RTCCameraVideoCapturer.h index 370bfa70f0..74f0b39925 100644 --- a/sdk/objc/components/capturer/RTCCameraVideoCapturer.h +++ b/sdk/objc/components/capturer/RTCCameraVideoCapturer.h @@ -30,6 +30,8 @@ NS_EXTENSION_UNAVAILABLE_IOS("Camera not available in app extensions.") // Returns list of formats that are supported by this class for this device. + (NSArray *)supportedFormatsForDevice:(AVCaptureDevice *)device; ++ (CGFloat)defaultZoomFactorForDeviceType:(AVCaptureDeviceType)deviceType; + // Returns the most efficient supported output pixel format for this capturer. - (FourCharCode)preferredOutputPixelFormat; diff --git a/sdk/objc/components/capturer/RTCCameraVideoCapturer.m b/sdk/objc/components/capturer/RTCCameraVideoCapturer.m index e7c47b4e99..26075aeca4 100644 --- a/sdk/objc/components/capturer/RTCCameraVideoCapturer.m +++ b/sdk/objc/components/capturer/RTCCameraVideoCapturer.m @@ -42,7 +42,7 @@ @implementation RTC_OBJC_TYPE (RTCCameraVideoCapturer) { FourCharCode _outputPixelFormat; RTCVideoRotation _rotation; #if TARGET_OS_IPHONE - UIDeviceOrientation _orientation; + UIInterfaceOrientation _orientation; BOOL _generatingOrientationNotifications; #endif } @@ -75,7 +75,7 @@ - (instancetype)initWithDelegate:(__weak id_generatingOrientationNotifications = YES; } + // Must be called on main + [self updateOrientation]; }); #endif @@ -168,8 +190,7 @@ - (void)startCaptureWithDevice:(AVCaptureDevice *)device NSError *error = nil; if (![self.currentDevice lockForConfiguration:&error]) { - RTCLogError(@"Failed to lock device %@. Error: %@", - self.currentDevice, + RTCLogError(@"Failed to lock device %@. Error: %@", self.currentDevice, error.userInfo); if (completionHandler) { completionHandler(error); @@ -178,11 +199,12 @@ - (void)startCaptureWithDevice:(AVCaptureDevice *)device return; } [self reconfigureCaptureSessionInput]; - [self updateOrientation]; [self updateDeviceCaptureFormat:format fps:fps]; [self updateVideoDataOutputPixelFormat:format]; - [self.captureSession startRunning]; + [self updateZoomFactor]; [self.currentDevice unlockForConfiguration]; + + [self.captureSession startRunning]; self.isRunning = YES; if (completionHandler) { completionHandler(nil); @@ -221,10 +243,7 @@ - (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHand #if TARGET_OS_IPHONE - (void)deviceOrientationDidChange:(NSNotification *)notification { - [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession - block:^{ - [self updateOrientation]; - }]; + [self updateOrientation]; } #endif @@ -260,22 +279,20 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position; } switch (_orientation) { - case UIDeviceOrientationPortrait: + case UIInterfaceOrientationPortrait: _rotation = RTCVideoRotation_90; break; - case UIDeviceOrientationPortraitUpsideDown: + case UIInterfaceOrientationPortraitUpsideDown: _rotation = RTCVideoRotation_270; break; - case UIDeviceOrientationLandscapeLeft: - _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0; - break; - case UIDeviceOrientationLandscapeRight: + case UIInterfaceOrientationLandscapeLeft: _rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180; break; - case UIDeviceOrientationFaceUp: - case UIDeviceOrientationFaceDown: - case UIDeviceOrientationUnknown: - // Ignore. + case UIInterfaceOrientationLandscapeRight: + _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0; + break; + case UIInterfaceOrientationUnknown: + _rotation = RTCVideoRotation_0; break; } #else @@ -286,7 +303,7 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer]; int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * - kNanosecondsPerSecond; + kNanosecondsPerSecond; RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer rotation:_rotation @@ -416,8 +433,7 @@ - (void)handleApplicationDidBecomeActive:(NSNotification *)notification { - (dispatch_queue_t)frameQueue { if (!_frameQueue) { _frameQueue = RTCDispatchQueueCreateWithTarget( - "org.webrtc.cameravideocapturer.video", - DISPATCH_QUEUE_SERIAL, + "org.webrtc.cameravideocapturer.video", DISPATCH_QUEUE_SERIAL, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0)); } return _frameQueue; @@ -490,19 +506,31 @@ - (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger @"updateDeviceCaptureFormat must be called on the capture queue."); @try { _currentDevice.activeFormat = format; - _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); + if (![NSStringFromClass([_currentDevice class]) isEqualToString:@"AVCaptureDALDevice"]) { + _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); + } } @catch (NSException *exception) { RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo); return; } } +- (void)updateZoomFactor { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], + @"updateZoomFactor must be called on the capture queue."); + +#if TARGET_OS_IOS || TARGET_OS_TV + CGFloat videoZoomFactor = [[self class] defaultZoomFactorForDeviceType:_currentDevice.deviceType]; + [_currentDevice setVideoZoomFactor:videoZoomFactor]; +#endif +} + - (void)reconfigureCaptureSessionInput { NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], @"reconfigureCaptureSessionInput must be called on the capture queue."); NSError *error = nil; - AVCaptureDeviceInput *input = - [AVCaptureDeviceInput deviceInputWithDevice:_currentDevice error:&error]; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:_currentDevice + error:&error]; if (!input) { RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription); return; @@ -519,12 +547,19 @@ - (void)reconfigureCaptureSessionInput { [_captureSession commitConfiguration]; } -- (void)updateOrientation { - NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], - @"updateOrientation must be called on the capture queue."); #if TARGET_OS_IPHONE - _orientation = [UIDevice currentDevice].orientation; -#endif +- (void)updateOrientation { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeMain], + @"statusBarOrientation must be called on the main queue."); + // statusBarOrientation must be called on the main queue + UIInterfaceOrientation newOrientation = [UIApplication sharedApplication].statusBarOrientation; + + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + // Must be called on the capture queue + self->_orientation = newOrientation; + }]; } +#endif @end diff --git a/sdk/objc/components/capturer/RTCDesktopCapturer+Private.h b/sdk/objc/components/capturer/RTCDesktopCapturer+Private.h new file mode 100644 index 0000000000..30aed69d29 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopCapturer+Private.h @@ -0,0 +1,49 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDesktopCapturer.h" + +#include "sdk/objc/native/src/objc_desktop_capture.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(DesktopCapturerDelegate) +-(void)didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *) frame; +-(void)didSourceCaptureStart; +-(void)didSourceCapturePaused; +-(void)didSourceCaptureStop; +-(void)didSourceCaptureError; +@end + +@interface RTC_OBJC_TYPE(RTCDesktopCapturer) () + +@property(nonatomic, readonly)std::shared_ptr nativeCapturer; + +- (void)didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +-(void)didSourceCaptureStart; + +-(void)didSourceCapturePaused; + +-(void)didSourceCaptureStop; + +-(void)didSourceCaptureError; + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopCapturer.h b/sdk/objc/components/capturer/RTCDesktopCapturer.h new file mode 100644 index 0000000000..b63912acf0 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopCapturer.h @@ -0,0 +1,61 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import + +#import "RTCMacros.h" +#import "RTCVideoCapturer.h" +#import "RTCDesktopSource.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCDesktopCapturer); + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCDesktopCapturerDelegate) +-(void)didSourceCaptureStart:(RTC_OBJC_TYPE(RTCDesktopCapturer) *) capturer; + +-(void)didSourceCapturePaused:(RTC_OBJC_TYPE(RTCDesktopCapturer) *) capturer; + +-(void)didSourceCaptureStop:(RTC_OBJC_TYPE(RTCDesktopCapturer) *) capturer; + +-(void)didSourceCaptureError:(RTC_OBJC_TYPE(RTCDesktopCapturer) *) capturer; +@end + +RTC_OBJC_EXPORT +// Screen capture that implements RTCVideoCapturer. Delivers frames to a +// RTCVideoCapturerDelegate (usually RTCVideoSource). +@interface RTC_OBJC_TYPE (RTCDesktopCapturer) : RTC_OBJC_TYPE(RTCVideoCapturer) + +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCDesktopSource) *source; + +- (instancetype)initWithSource:(RTC_OBJC_TYPE(RTCDesktopSource) *)source delegate:(__weak id)delegate captureDelegate:(__weak id)captureDelegate; + +- (instancetype)initWithDefaultScreen:(__weak id)delegate captureDelegate:(__weak id)captureDelegate; + +- (void)startCapture; + +- (void)startCaptureWithFPS:(NSInteger)fps; + +- (void)stopCapture; + +- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/capturer/RTCDesktopCapturer.mm b/sdk/objc/components/capturer/RTCDesktopCapturer.mm new file mode 100644 index 0000000000..a1948684d3 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopCapturer.mm @@ -0,0 +1,104 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "base/RTCLogging.h" +#import "base/RTCVideoFrameBuffer.h" + +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#import "RTCDesktopCapturer.h" +#import "RTCDesktopCapturer+Private.h" +#import "RTCDesktopSource+Private.h" + +@implementation RTC_OBJC_TYPE (RTCDesktopCapturer) { + __weak id _delegate; +} + +@synthesize nativeCapturer = _nativeCapturer; +@synthesize source = _source; + +- (instancetype)initWithSource:(RTC_OBJC_TYPE(RTCDesktopSource) *)source delegate:(__weak id)delegate captureDelegate:(__weak id)captureDelegate { + if (self = [super initWithDelegate:captureDelegate]) { + webrtc::DesktopType captureType = webrtc::kScreen; + if(source.sourceType == RTCDesktopSourceTypeWindow) { + captureType = webrtc::kWindow; + } + _nativeCapturer = std::make_shared(captureType, source.nativeMediaSource->id(), self); + _source = source; + _delegate = delegate; + } + return self; +} + +- (instancetype)initWithDefaultScreen:(__weak id)delegate captureDelegate:(__weak id)captureDelegate { + if (self = [super initWithDelegate:captureDelegate]) { + _nativeCapturer = std::make_unique(webrtc::kScreen, -1, self); + _source = nil; + _delegate = delegate; + } + return self; +} + + +-(void)dealloc { + _nativeCapturer->Stop(); + _nativeCapturer = nullptr; +} + +- (void)startCapture { + [self didSourceCaptureStart]; + _nativeCapturer->Start(30); +} + +- (void)startCaptureWithFPS:(NSInteger)fps { + _nativeCapturer->Start(fps); +} + +- (void)didCaptureVideoFrame + : (RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + [self.delegate capturer:self didCaptureVideoFrame:frame]; +} + +- (void)stopCapture { + _nativeCapturer->Stop(); +} + +- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler { + [self stopCapture]; + if(completionHandler != nil) { + completionHandler(); + } +} + +-(void)didSourceCaptureStart { + [_delegate didSourceCaptureStart:self]; +} + +-(void)didSourceCapturePaused { + [_delegate didSourceCapturePaused:self]; +} + +-(void)didSourceCaptureStop { + [_delegate didSourceCaptureStop:self]; +} + +-(void)didSourceCaptureError { + [_delegate didSourceCaptureError:self]; +} + +@end diff --git a/sdk/objc/components/capturer/RTCDesktopMediaList+Private.h b/sdk/objc/components/capturer/RTCDesktopMediaList+Private.h new file mode 100644 index 0000000000..eb1e76ddbb --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopMediaList+Private.h @@ -0,0 +1,40 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDesktopMediaList.h" + +namespace webrtc { + class ObjCDesktopMediaList; + class MediaSource; +} + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE(RTCDesktopMediaList) () + +@property(nonatomic, readonly)std::shared_ptr nativeMediaList; + +-(void)mediaSourceAdded:(webrtc::MediaSource *) source; + +-(void)mediaSourceRemoved:(webrtc::MediaSource *) source; + +-(void)mediaSourceNameChanged:(webrtc::MediaSource *) source; + +-(void)mediaSourceThumbnailChanged:(webrtc::MediaSource *) source; + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopMediaList.h b/sdk/objc/components/capturer/RTCDesktopMediaList.h new file mode 100644 index 0000000000..fafeaf5e0d --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopMediaList.h @@ -0,0 +1,51 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import + +#import "RTCMacros.h" +#import "RTCDesktopSource.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCDesktopMediaListDelegate) + +- (void)didDesktopSourceAdded:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; + +- (void)didDesktopSourceRemoved:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; + +- (void)didDesktopSourceNameChanged:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; + +- (void)didDesktopSourceThumbnailChanged:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDesktopMediaList) : NSObject + +-(instancetype)initWithType:(RTCDesktopSourceType)type delegate:(__weak id)delegate; + +@property(nonatomic, readonly) RTCDesktopSourceType sourceType; + +- (int32_t)UpdateSourceList:(BOOL)forceReload updateAllThumbnails:(BOOL)updateThumbnail; + +- (NSArray*) getSources; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/capturer/RTCDesktopMediaList.mm b/sdk/objc/components/capturer/RTCDesktopMediaList.mm new file mode 100644 index 0000000000..2bd6c1da0e --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopMediaList.mm @@ -0,0 +1,99 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDesktopMediaList.h" + +#import "RTCDesktopSource+Private.h" +#import "RTCDesktopMediaList+Private.h" + +@implementation RTC_OBJC_TYPE(RTCDesktopMediaList) { + RTCDesktopSourceType _sourceType; + NSMutableArray* _sources; + __weak id _delegate; +} + +@synthesize sourceType = _sourceType; +@synthesize nativeMediaList = _nativeMediaList; + +- (instancetype)initWithType:(RTCDesktopSourceType)type delegate:(__weak id)delegate{ + if (self = [super init]) { + webrtc::DesktopType captureType = webrtc::kScreen; + if(type == RTCDesktopSourceTypeWindow) { + captureType = webrtc::kWindow; + } + _nativeMediaList = std::make_shared(captureType, self); + _sourceType = type; + _delegate = delegate; + } + return self; +} + +- (int32_t)UpdateSourceList:(BOOL)forceReload updateAllThumbnails:(BOOL)updateThumbnail { + return _nativeMediaList->UpdateSourceList(forceReload, updateThumbnail); +} + +-(NSArray*) getSources { + _sources = [NSMutableArray array]; + int sourceCount = _nativeMediaList->GetSourceCount(); + for (int i = 0; i < sourceCount; i++) { + webrtc::MediaSource *mediaSource = _nativeMediaList->GetSource(i); + [_sources addObject:[[RTC_OBJC_TYPE(RTCDesktopSource) alloc] initWithNativeSource:mediaSource sourceType:_sourceType]]; + } + return _sources; +} + +-(void)mediaSourceAdded:(webrtc::MediaSource *) source { + RTC_OBJC_TYPE(RTCDesktopSource) *desktopSource = [[RTC_OBJC_TYPE(RTCDesktopSource) alloc] initWithNativeSource:source sourceType:_sourceType]; + [_sources addObject:desktopSource]; + [_delegate didDesktopSourceAdded:desktopSource]; +} + +-(void)mediaSourceRemoved:(webrtc::MediaSource *) source { + RTC_OBJC_TYPE(RTCDesktopSource) *desktopSource = [self getSourceById:source]; + if(desktopSource != nil) { + [_sources removeObject:desktopSource]; + [_delegate didDesktopSourceRemoved:desktopSource]; + } +} + +-(void)mediaSourceNameChanged:(webrtc::MediaSource *) source { + RTC_OBJC_TYPE(RTCDesktopSource) *desktopSource = [self getSourceById:source]; + if(desktopSource != nil) { + [desktopSource setName:source->name().c_str()]; + [_delegate didDesktopSourceNameChanged:desktopSource]; + } +} + +-(void)mediaSourceThumbnailChanged:(webrtc::MediaSource *) source { + RTC_OBJC_TYPE(RTCDesktopSource) *desktopSource = [self getSourceById:source]; + if(desktopSource != nil) { + [desktopSource setThumbnail:source->thumbnail()]; + [_delegate didDesktopSourceThumbnailChanged:desktopSource]; + } +} + +-(RTC_OBJC_TYPE(RTCDesktopSource) *)getSourceById:(webrtc::MediaSource *) source { + NSEnumerator *enumerator = [_sources objectEnumerator]; + RTC_OBJC_TYPE(RTCDesktopSource) *object; + while ((object = enumerator.nextObject) != nil) { + if(object.nativeMediaSource == source) { + return object; + } + } + return nil; +} + +@end \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopSource+Private.h b/sdk/objc/components/capturer/RTCDesktopSource+Private.h new file mode 100644 index 0000000000..3f4c4ef25f --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopSource+Private.h @@ -0,0 +1,37 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#import + +#import "RTCDesktopSource.h" + +#include "sdk/objc/native/src/objc_desktop_media_list.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE(RTCDesktopSource) () + +- (instancetype)initWithNativeSource:(webrtc::MediaSource*) nativeSource + sourceType:(RTCDesktopSourceType) sourceType; + +@property(nonatomic, readonly)webrtc::MediaSource* nativeMediaSource; + +-(void) setName:(const char *) name; + +-(void) setThumbnail:(std::vector) thumbnail; + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopSource.h b/sdk/objc/components/capturer/RTCDesktopSource.h new file mode 100644 index 0000000000..82da458ce6 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopSource.h @@ -0,0 +1,40 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#import +#import +#import + +#import "RTCMacros.h" + +typedef NS_ENUM(NSInteger, RTCDesktopSourceType) { + RTCDesktopSourceTypeScreen, + RTCDesktopSourceTypeWindow, +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDesktopSource) : NSObject + +@property(nonatomic, readonly) NSString *sourceId; + +@property(nonatomic, readonly) NSString *name; + +@property(nonatomic, readonly) NSImage *thumbnail; + +@property(nonatomic, readonly) RTCDesktopSourceType sourceType; + +-( NSImage *)UpdateThumbnail; + +@end \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopSource.mm b/sdk/objc/components/capturer/RTCDesktopSource.mm new file mode 100644 index 0000000000..e1bdc6893a --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopSource.mm @@ -0,0 +1,68 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCDesktopSource.h" +#import "RTCDesktopSource+Private.h" + +@implementation RTC_OBJC_TYPE(RTCDesktopSource) { + NSString *_sourceId; + NSString *_name; + NSImage *_thumbnail; + RTCDesktopSourceType _sourceType; +} + +@synthesize sourceId = _sourceId; +@synthesize name = _name; +@synthesize thumbnail = _thumbnail; +@synthesize sourceType = _sourceType; +@synthesize nativeMediaSource = _nativeMediaSource; + +- (instancetype)initWithNativeSource:(webrtc::MediaSource*)nativeSource + sourceType:(RTCDesktopSourceType) sourceType { + if (self = [super init]) { + _nativeMediaSource = nativeSource; + _sourceId = [NSString stringWithUTF8String:std::to_string(nativeSource->id()).c_str()]; + _name = [NSString stringWithUTF8String:nativeSource->name().c_str()]; + _thumbnail = [self createThumbnailFromNativeSource:nativeSource->thumbnail()]; + _sourceType = sourceType; + } + return self; +} + +-(NSImage*)createThumbnailFromNativeSource:(std::vector)thumbnail { + NSData* data = [[NSData alloc] initWithBytes:thumbnail.data() length:thumbnail.size()]; + NSImage *image = [[NSImage alloc] initWithData:data]; + return image; +} + +-( NSImage *)UpdateThumbnail { + if(_nativeMediaSource->UpdateThumbnail()) { + _thumbnail = [self createThumbnailFromNativeSource:_nativeMediaSource->thumbnail()]; + } + return _thumbnail; +} + +-(void)setName:(const char *) name { + _name = [NSString stringWithUTF8String:name]; +} + +-(void)setThumbnail:(std::vector) thumbnail { + _thumbnail = [self createThumbnailFromNativeSource:thumbnail]; +} + +@end diff --git a/sdk/objc/components/network/RTCNetworkMonitor+Private.h b/sdk/objc/components/network/RTCNetworkMonitor+Private.h index b5c786be18..f3761f7ba3 100644 --- a/sdk/objc/components/network/RTCNetworkMonitor+Private.h +++ b/sdk/objc/components/network/RTCNetworkMonitor+Private.h @@ -9,16 +9,18 @@ */ #import "RTCNetworkMonitor.h" +#import "RTCMacros.h" #include "sdk/objc/native/src/network_monitor_observer.h" -@interface RTCNetworkMonitor () +@interface RTC_OBJC_TYPE (RTCNetworkMonitor) +() -/** `observer` is a raw pointer and should be kept alive - * for this object's lifetime. - */ -- (instancetype)initWithObserver:(webrtc::NetworkMonitorObserver *)observer - NS_DESIGNATED_INITIALIZER; + /** `observer` is a raw pointer and should be kept alive + * for this object's lifetime. + */ + - (instancetype)initWithObserver + : (webrtc::NetworkMonitorObserver *)observer NS_DESIGNATED_INITIALIZER; /** Stops the receiver from posting updates to `observer`. */ - (void)stop; diff --git a/sdk/objc/components/network/RTCNetworkMonitor.h b/sdk/objc/components/network/RTCNetworkMonitor.h index 21d22f5463..4b0cb4baf0 100644 --- a/sdk/objc/components/network/RTCNetworkMonitor.h +++ b/sdk/objc/components/network/RTCNetworkMonitor.h @@ -10,12 +10,14 @@ #import +#import "RTCMacros.h" + NS_ASSUME_NONNULL_BEGIN /** Listens for NWPathMonitor updates and forwards the results to a C++ * observer. */ -@interface RTCNetworkMonitor : NSObject +@interface RTC_OBJC_TYPE (RTCNetworkMonitor): NSObject - (instancetype)init NS_UNAVAILABLE; diff --git a/sdk/objc/components/network/RTCNetworkMonitor.mm b/sdk/objc/components/network/RTCNetworkMonitor.mm index 7e75b2b4c0..2e42ab5290 100644 --- a/sdk/objc/components/network/RTCNetworkMonitor.mm +++ b/sdk/objc/components/network/RTCNetworkMonitor.mm @@ -46,7 +46,7 @@ } // namespace -@implementation RTCNetworkMonitor { +@implementation RTC_OBJC_TYPE (RTCNetworkMonitor) { webrtc::NetworkMonitorObserver *_observer; nw_path_monitor_t _pathMonitor; dispatch_queue_t _monitorQueue; @@ -63,12 +63,12 @@ - (instancetype)initWithObserver:(webrtc::NetworkMonitorObserver *)observer { return nil; } RTCLog(@"NW path monitor created."); - __weak RTCNetworkMonitor *weakSelf = self; + __weak RTC_OBJC_TYPE(RTCNetworkMonitor) *weakSelf = self; nw_path_monitor_set_update_handler(_pathMonitor, ^(nw_path_t path) { if (weakSelf == nil) { return; } - RTCNetworkMonitor *strongSelf = weakSelf; + RTC_OBJC_TYPE(RTCNetworkMonitor) *strongSelf = weakSelf; RTCLog(@"NW path monitor: updated."); nw_path_status_t status = nw_path_get_status(path); if (status == nw_path_status_invalid) { diff --git a/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h b/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h index e5987fe22a..c4e2724042 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h +++ b/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h @@ -13,5 +13,5 @@ #import "RTCMTLRenderer.h" NS_AVAILABLE(10_11, 9_0) -@interface RTCMTLI420Renderer : RTCMTLRenderer +@interface RTC_OBJC_TYPE(RTCMTLI420Renderer): RTC_OBJC_TYPE(RTCMTLRenderer) @end diff --git a/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm b/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm index eba8800240..d7852ad958 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm +++ b/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm @@ -70,7 +70,7 @@ fragment half4 fragmentColorConversion( return half4(out); }); -@implementation RTCMTLI420Renderer { +@implementation RTC_OBJC_TYPE(RTCMTLI420Renderer) { // Textures. id _yTexture; id _uTexture; diff --git a/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m b/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m deleted file mode 100644 index 625fb1caa7..0000000000 --- a/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "RTCMTLNSVideoView.h" - -#import -#import - -#import "base/RTCVideoFrame.h" - -#import "RTCMTLI420Renderer.h" - -@interface RTC_OBJC_TYPE (RTCMTLNSVideoView) -() @property(nonatomic) id renderer; -@property(nonatomic, strong) MTKView *metalView; -@property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; -@end - -@implementation RTC_OBJC_TYPE (RTCMTLNSVideoView) { - id _renderer; -} - -@synthesize delegate = _delegate; -@synthesize renderer = _renderer; -@synthesize metalView = _metalView; -@synthesize videoFrame = _videoFrame; - -- (instancetype)initWithFrame:(CGRect)frameRect { - self = [super initWithFrame:frameRect]; - if (self) { - [self configure]; - } - return self; -} - -- (instancetype)initWithCoder:(NSCoder *)aCoder { - self = [super initWithCoder:aCoder]; - if (self) { - [self configure]; - } - return self; -} - -#pragma mark - Private - -+ (BOOL)isMetalAvailable { - return [MTLCopyAllDevices() count] > 0; -} - -- (void)configure { - if ([[self class] isMetalAvailable]) { - _metalView = [[MTKView alloc] initWithFrame:self.bounds]; - [self addSubview:_metalView]; - _metalView.layerContentsPlacement = NSViewLayerContentsPlacementScaleProportionallyToFit; - _metalView.translatesAutoresizingMaskIntoConstraints = NO; - _metalView.framebufferOnly = YES; - _metalView.delegate = self; - - _renderer = [[RTCMTLI420Renderer alloc] init]; - if (![(RTCMTLI420Renderer *)_renderer addRenderingDestination:_metalView]) { - _renderer = nil; - }; - } -} - -- (void)updateConstraints { - NSDictionary *views = NSDictionaryOfVariableBindings(_metalView); - - NSArray *constraintsHorizontal = - [NSLayoutConstraint constraintsWithVisualFormat:@"H:|-0-[_metalView]-0-|" - options:0 - metrics:nil - views:views]; - [self addConstraints:constraintsHorizontal]; - - NSArray *constraintsVertical = - [NSLayoutConstraint constraintsWithVisualFormat:@"V:|-0-[_metalView]-0-|" - options:0 - metrics:nil - views:views]; - [self addConstraints:constraintsVertical]; - [super updateConstraints]; -} - -#pragma mark - MTKViewDelegate methods -- (void)drawInMTKView:(nonnull MTKView *)view { - if (self.videoFrame == nil) { - return; - } - if (view == self.metalView) { - [_renderer drawFrame:self.videoFrame]; - } -} - -- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { -} - -#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) - -- (void)setSize:(CGSize)size { - _metalView.drawableSize = size; - dispatch_async(dispatch_get_main_queue(), ^{ - [self.delegate videoView:self didChangeVideoSize:size]; - }); - [_metalView draw]; -} - -- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { - if (frame == nil) { - return; - } - self.videoFrame = [frame newI420VideoFrame]; -} - -@end diff --git a/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h b/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h index 866b7ea17e..125612a269 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h +++ b/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h @@ -13,6 +13,6 @@ #import "RTCMTLRenderer.h" NS_AVAILABLE(10_11, 9_0) -@interface RTCMTLNV12Renderer : RTCMTLRenderer +@interface RTC_OBJC_TYPE(RTCMTLNV12Renderer): RTC_OBJC_TYPE(RTCMTLRenderer) @end diff --git a/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm b/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm index 7b037c6dbc..c4000b1b1d 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm +++ b/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm @@ -60,7 +60,7 @@ fragment half4 fragmentColorConversion( return half4(out); }); -@implementation RTCMTLNV12Renderer { +@implementation RTC_OBJC_TYPE(RTCMTLNV12Renderer) { // Textures. CVMetalTextureCacheRef _textureCache; id _yTexture; diff --git a/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h b/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h index 9db422cd22..5e355a8504 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h +++ b/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h @@ -11,12 +11,13 @@ #import #import "RTCMTLRenderer.h" +#import "RTCMacros.h" /** @abstract RGB/BGR renderer. * @discussion This renderer handles both kCVPixelFormatType_32BGRA and * kCVPixelFormatType_32ARGB. */ NS_AVAILABLE(10_11, 9_0) -@interface RTCMTLRGBRenderer : RTCMTLRenderer +@interface RTC_OBJC_TYPE (RTCMTLRGBRenderer): RTC_OBJC_TYPE(RTCMTLRenderer) @end diff --git a/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm b/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm index e5dc4ef80a..6ca4a4000d 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm +++ b/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm @@ -30,12 +30,12 @@ } Vertex; typedef struct { - float4 position[[position]]; + float4 position [[position]]; float2 texcoord; } VertexIO; - vertex VertexIO vertexPassthrough(constant Vertex *verticies[[buffer(0)]], - uint vid[[vertex_id]]) { + vertex VertexIO vertexPassthrough(constant Vertex * verticies [[buffer(0)]], + uint vid [[vertex_id]]) { VertexIO out; constant Vertex &v = verticies[vid]; out.position = float4(float2(v.position), 0.0, 1.0); @@ -43,9 +43,9 @@ vertex VertexIO vertexPassthrough(constant Vertex *verticies[[buffer(0)]], return out; } - fragment half4 fragmentColorConversion(VertexIO in[[stage_in]], - texture2d texture[[texture(0)]], - constant bool &isARGB[[buffer(0)]]) { + fragment half4 fragmentColorConversion(VertexIO in [[stage_in]], + texture2d texture [[texture(0)]], + constant bool &isARGB [[buffer(0)]]) { constexpr sampler s(address::clamp_to_edge, filter::linear); half4 out = texture.sample(s, in.texcoord); @@ -56,7 +56,7 @@ fragment half4 fragmentColorConversion(VertexIO in[[stage_in]], return out; }); -@implementation RTCMTLRGBRenderer { +@implementation RTC_OBJC_TYPE (RTCMTLRGBRenderer) { // Textures. CVMetalTextureCacheRef _textureCache; id _texture; @@ -73,8 +73,8 @@ - (BOOL)addRenderingDestination:(__kindof MTKView *)view { } - (BOOL)initializeTextureCache { - CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice], - nil, &_textureCache); + CVReturn status = CVMetalTextureCacheCreate( + kCFAllocatorDefault, nil, [self currentMetalDevice], nil, &_textureCache); if (status != kCVReturnSuccess) { RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status); return NO; @@ -130,9 +130,15 @@ - (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { return NO; } - CVReturn result = CVMetalTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, _textureCache, pixelBuffer, nil, mtlPixelFormat, - width, height, 0, &textureOut); + CVReturn result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + _textureCache, + pixelBuffer, + nil, + mtlPixelFormat, + width, + height, + 0, + &textureOut); if (result == kCVReturnSuccess) { gpuTexture = CVMetalTextureGetTexture(textureOut); } diff --git a/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h b/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h index 916d4d4430..f6a82db56a 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h +++ b/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h @@ -16,7 +16,7 @@ NS_ASSUME_NONNULL_BEGIN -@interface RTCMTLRenderer (Private) +@interface RTC_OBJC_TYPE(RTCMTLRenderer) (Private) - (nullable id)currentMetalDevice; - (NSString *)shaderSource; - (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame; diff --git a/sdk/objc/components/renderer/metal/RTCMTLRenderer.h b/sdk/objc/components/renderer/metal/RTCMTLRenderer.h index aa31545973..6bbca3d985 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLRenderer.h +++ b/sdk/objc/components/renderer/metal/RTCMTLRenderer.h @@ -21,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN /** * Protocol defining ability to render RTCVideoFrame in Metal enabled views. */ -@protocol RTCMTLRenderer +@protocol RTC_OBJC_TYPE(RTCMTLRenderer) /** * Method to be implemented to perform actual rendering of the provided frame. @@ -49,7 +49,7 @@ NS_ASSUME_NONNULL_BEGIN * Implementation of RTCMTLRenderer protocol. */ NS_AVAILABLE(10_11, 9_0) -@interface RTCMTLRenderer : NSObject +@interface RTC_OBJC_TYPE(RTCMTLRenderer) : NSObject /** @abstract A wrapped RTCVideoRotation, or nil. @discussion When not nil, the rotation of the actual frame is ignored when rendering. diff --git a/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm b/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm index 410590a7b1..ca3fcc3e51 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm +++ b/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm @@ -87,7 +87,7 @@ static inline void getCubeVertexData(int cropX, // In future we might use triple buffering method if it improves performance. static const NSInteger kMaxInflightBuffers = 1; -@implementation RTCMTLRenderer { +@implementation RTC_OBJC_TYPE(RTCMTLRenderer) { __kindof MTKView *_view; // Controller. diff --git a/sdk/objc/components/renderer/metal/RTCMTLVideoView.h b/sdk/objc/components/renderer/metal/RTCMTLVideoView.h index 3320d12076..bed02ffa92 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLVideoView.h +++ b/sdk/objc/components/renderer/metal/RTCMTLVideoView.h @@ -10,6 +10,10 @@ #import +#if TARGET_OS_OSX +#import +#endif + #import "RTCMacros.h" #import "RTCVideoFrame.h" #import "RTCVideoRenderer.h" @@ -22,14 +26,26 @@ NS_ASSUME_NONNULL_BEGIN * It has id property that renders video frames in the view's * bounds using Metal. */ +#if TARGET_OS_IPHONE NS_CLASS_AVAILABLE_IOS(9) +#elif TARGET_OS_OSX +NS_AVAILABLE_MAC(10.11) +#endif RTC_OBJC_EXPORT -@interface RTC_OBJC_TYPE (RTCMTLVideoView) : UIView +@interface RTC_OBJC_TYPE (RTCMTLVideoView) : + +#if TARGET_OS_IPHONE + UIView +#elif TARGET_OS_OSX + NSView +#endif @property(nonatomic, weak) id delegate; +#if TARGET_OS_IPHONE @property(nonatomic) UIViewContentMode videoContentMode; +#endif /** @abstract Enables/disables rendering. */ @@ -39,6 +55,8 @@ RTC_OBJC_EXPORT */ @property(nonatomic, nullable) NSValue* rotationOverride; ++ (BOOL)isMetalAvailable; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/renderer/metal/RTCMTLVideoView.m b/sdk/objc/components/renderer/metal/RTCMTLVideoView.m index c5d9e4385f..d4d98a0bf4 100644 --- a/sdk/objc/components/renderer/metal/RTCMTLVideoView.m +++ b/sdk/objc/components/renderer/metal/RTCMTLVideoView.m @@ -22,17 +22,12 @@ #import "RTCMTLNV12Renderer.h" #import "RTCMTLRGBRenderer.h" -// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime. -// Linking errors occur when compiling for architectures that don't support Metal. -#define MTKViewClass NSClassFromString(@"MTKView") -#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer") -#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer") -#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer") - -@interface RTC_OBJC_TYPE (RTCMTLVideoView) -() @property(nonatomic) RTCMTLI420Renderer *rendererI420; -@property(nonatomic) RTCMTLNV12Renderer *rendererNV12; -@property(nonatomic) RTCMTLRGBRenderer *rendererRGB; +#import "RTCMTLRenderer+Private.h" + +@interface RTC_OBJC_TYPE (RTCMTLVideoView) () +@property(nonatomic) RTC_OBJC_TYPE(RTCMTLI420Renderer) *rendererI420; +@property(nonatomic) RTC_OBJC_TYPE(RTCMTLNV12Renderer) * rendererNV12; +@property(nonatomic) RTC_OBJC_TYPE(RTCMTLRGBRenderer) * rendererRGB; @property(nonatomic) MTKView *metalView; @property(atomic) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; @property(nonatomic) CGSize videoFrameSize; @@ -51,6 +46,14 @@ @implementation RTC_OBJC_TYPE (RTCMTLVideoView) @synthesize lastFrameTimeNs = _lastFrameTimeNs; @synthesize rotationOverride = _rotationOverride; ++ (BOOL)isMetalAvailable { +#if TARGET_OS_IPHONE + return MTLCreateSystemDefaultDevice() != nil; +#elif TARGET_OS_OSX + return [MTLCopyAllDevices() count] > 0; +#endif +} + - (instancetype)initWithFrame:(CGRect)frameRect { self = [super initWithFrame:frameRect]; if (self) { @@ -75,6 +78,7 @@ - (void)setEnabled:(BOOL)enabled { self.metalView.paused = !enabled; } +#if TARGET_OS_IPHONE - (UIViewContentMode)videoContentMode { return self.metalView.contentMode; } @@ -82,27 +86,24 @@ - (UIViewContentMode)videoContentMode { - (void)setVideoContentMode:(UIViewContentMode)mode { self.metalView.contentMode = mode; } +#endif #pragma mark - Private -+ (BOOL)isMetalAvailable { - return MTLCreateSystemDefaultDevice() != nil; -} - + (MTKView *)createMetalView:(CGRect)frame { - return [[MTKViewClass alloc] initWithFrame:frame]; + return [[MTKView alloc] initWithFrame:frame]; } -+ (RTCMTLNV12Renderer *)createNV12Renderer { - return [[RTCMTLNV12RendererClass alloc] init]; ++ (RTC_OBJC_TYPE(RTCMTLNV12Renderer) *)createNV12Renderer { + return [[RTC_OBJC_TYPE(RTCMTLNV12Renderer) alloc] init]; } -+ (RTCMTLI420Renderer *)createI420Renderer { - return [[RTCMTLI420RendererClass alloc] init]; ++ (RTC_OBJC_TYPE(RTCMTLI420Renderer) *)createI420Renderer { + return [[RTC_OBJC_TYPE(RTCMTLI420Renderer) alloc] init]; } -+ (RTCMTLRGBRenderer *)createRGBRenderer { - return [[RTCMTLRGBRenderer alloc] init]; ++ (RTC_OBJC_TYPE(RTCMTLRGBRenderer) *)createRGBRenderer { + return [[RTC_OBJC_TYPE(RTCMTLRGBRenderer) alloc] init]; } - (void)configure { @@ -111,19 +112,24 @@ - (void)configure { self.metalView = [RTC_OBJC_TYPE(RTCMTLVideoView) createMetalView:self.bounds]; self.metalView.delegate = self; +#if TARGET_OS_IPHONE self.metalView.contentMode = UIViewContentModeScaleAspectFill; +#elif TARGET_OS_OSX + self.metalView.layerContentsPlacement = NSViewLayerContentsPlacementScaleProportionallyToFit; +#endif + [self addSubview:self.metalView]; self.videoFrameSize = CGSizeZero; } +#if TARGET_OS_IPHONE - (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { - [super setMultipleTouchEnabled:multipleTouchEnabled]; - self.metalView.multipleTouchEnabled = multipleTouchEnabled; + [super setMultipleTouchEnabled:multipleTouchEnabled]; + self.metalView.multipleTouchEnabled = multipleTouchEnabled; } +#endif -- (void)layoutSubviews { - [super layoutSubviews]; - +- (void)performLayout { CGRect bounds = self.bounds; self.metalView.frame = bounds; if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) { @@ -148,7 +154,7 @@ - (void)drawInMTKView:(nonnull MTKView *)view { return; } - RTCMTLRenderer *renderer; + RTC_OBJC_TYPE(RTCMTLRenderer) * renderer; if ([videoFrame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)videoFrame.buffer; const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer); @@ -203,10 +209,10 @@ - (void)setRotationOverride:(NSValue *)rotationOverride { [self setNeedsLayout]; } -- (RTCVideoRotation)frameRotation { +- (RTCVideoRotation)videoRotation { if (self.rotationOverride) { RTCVideoRotation rotation; - if (@available(iOS 11, *)) { + if (@available(iOS 11, macos 10.13, *)) { [self.rotationOverride getValue:&rotation size:sizeof(rotation)]; } else { [self.rotationOverride getValue:&rotation]; @@ -220,10 +226,10 @@ - (RTCVideoRotation)frameRotation { - (CGSize)drawableSize { // Flip width/height if the rotations are not the same. CGSize videoFrameSize = self.videoFrameSize; - RTCVideoRotation frameRotation = [self frameRotation]; + RTCVideoRotation videoRotation = [self videoRotation]; BOOL useLandscape = - (frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180); + (videoRotation == RTCVideoRotation_0) || (videoRotation == RTCVideoRotation_180); BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) || (self.videoFrame.rotation == RTCVideoRotation_180); @@ -259,7 +265,34 @@ - (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { RTCLogInfo(@"Incoming frame is nil. Exiting render callback."); return; } - self.videoFrame = frame; + + // Workaround to support RTCCVPixelBuffer rendering. + // RTCMTLRGBRenderer seems to be broken at the moment. + BOOL useI420 = NO; + if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer); + useI420 = pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB; + } + self.videoFrame = useI420 ? [frame newI420VideoFrame] : frame; +} + +#pragma mark - Cross platform + +#if TARGET_OS_IPHONE +- (void)layoutSubviews { + [super layoutSubviews]; + [self performLayout]; +} +#elif TARGET_OS_OSX +- (void)layout { + [super layout]; + [self performLayout]; +} + +- (void)setNeedsLayout { + self.needsLayout = YES; } +#endif @end diff --git a/sdk/objc/components/renderer/opengl/RTCDefaultShader.h b/sdk/objc/components/renderer/opengl/RTCDefaultShader.h index 71a073ab21..b00cf8047d 100644 --- a/sdk/objc/components/renderer/opengl/RTCDefaultShader.h +++ b/sdk/objc/components/renderer/opengl/RTCDefaultShader.h @@ -16,7 +16,7 @@ NS_ASSUME_NONNULL_BEGIN * and RTCEAGLVideoView if no external shader is specified. This shader will render * the video in a rectangle without any color or geometric transformations. */ -@interface RTCDefaultShader : NSObject +@interface RTC_OBJC_TYPE(RTCDefaultShader) : NSObject @end diff --git a/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm b/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm index 9d686f625c..7f9373dc3a 100644 --- a/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm +++ b/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm @@ -65,7 +65,7 @@ " 1.0);\n" " }\n"; -@implementation RTCDefaultShader { +@implementation RTC_OBJC_TYPE(RTCDefaultShader) { GLuint _vertexBuffer; GLuint _vertexArray; // Store current rotation and only upload new vertex data when rotation changes. diff --git a/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h b/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h index b78501e9e6..1c5b64fdfc 100644 --- a/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h +++ b/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h @@ -10,11 +10,13 @@ #import +#import "RTCMacros.h" + // RTCDisplayLinkTimer wraps a CADisplayLink and is set to fire every two screen // refreshes, which should be 30fps. We wrap the display link in order to avoid // a retain cycle since CADisplayLink takes a strong reference onto its target. // The timer is paused by default. -@interface RTCDisplayLinkTimer : NSObject +@interface RTC_OBJC_TYPE (RTCDisplayLinkTimer): NSObject @property(nonatomic) BOOL isPaused; diff --git a/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m b/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m index 906bb898d6..f4cf03304d 100644 --- a/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m +++ b/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m @@ -12,7 +12,7 @@ #import -@implementation RTCDisplayLinkTimer { +@implementation RTC_OBJC_TYPE (RTCDisplayLinkTimer) { CADisplayLink *_displayLink; void (^_timerHandler)(void); } @@ -21,17 +21,15 @@ - (instancetype)initWithTimerHandler:(void (^)(void))timerHandler { NSParameterAssert(timerHandler); if (self = [super init]) { _timerHandler = timerHandler; - _displayLink = - [CADisplayLink displayLinkWithTarget:self - selector:@selector(displayLinkDidFire:)]; + _displayLink = [CADisplayLink displayLinkWithTarget:self + selector:@selector(displayLinkDidFire:)]; _displayLink.paused = YES; #if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_10_0 _displayLink.preferredFramesPerSecond = 30; #else [_displayLink setFrameInterval:2]; #endif - [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] - forMode:NSRunLoopCommonModes]; + [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes]; } return self; } diff --git a/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m b/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m index 89e62d2ce7..0a00494d2d 100644 --- a/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m +++ b/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m @@ -42,14 +42,14 @@ @interface RTC_OBJC_TYPE (RTCEAGLVideoView) @end @implementation RTC_OBJC_TYPE (RTCEAGLVideoView) { - RTCDisplayLinkTimer *_timer; + RTC_OBJC_TYPE(RTCDisplayLinkTimer) * _timer; EAGLContext *_glContext; // This flag should only be set and read on the main thread (e.g. by // setNeedsDisplay) BOOL _isDirty; id _shader; - RTCNV12TextureCache *_nv12TextureCache; - RTCI420TextureCache *_i420TextureCache; + RTC_OBJC_TYPE(RTCNV12TextureCache) *_nv12TextureCache; + RTC_OBJC_TYPE(RTCI420TextureCache) *_i420TextureCache; // As timestamps should be unique between frames, will store last // drawn frame timestamp instead of the whole frame to reduce memory usage. int64_t _lastDrawnFrameTimeStampNs; @@ -61,11 +61,11 @@ @implementation RTC_OBJC_TYPE (RTCEAGLVideoView) { @synthesize rotationOverride = _rotationOverride; - (instancetype)initWithFrame:(CGRect)frame { - return [self initWithFrame:frame shader:[[RTCDefaultShader alloc] init]]; + return [self initWithFrame:frame shader:[[RTC_OBJC_TYPE(RTCDefaultShader) alloc] init]]; } - (instancetype)initWithCoder:(NSCoder *)aDecoder { - return [self initWithCoder:aDecoder shader:[[RTCDefaultShader alloc] init]]; + return [self initWithCoder:aDecoder shader:[[RTC_OBJC_TYPE(RTCDefaultShader) alloc] init]]; } - (instancetype)initWithFrame:(CGRect)frame shader:(id)shader { @@ -90,8 +90,7 @@ - (instancetype)initWithCoder:(NSCoder *)aDecoder } - (BOOL)configure { - EAGLContext *glContext = - [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; + EAGLContext *glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; if (!glContext) { glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2]; } @@ -102,8 +101,7 @@ - (BOOL)configure { _glContext = glContext; // GLKView manages a framebuffer for us. - _glkView = [[GLKView alloc] initWithFrame:CGRectZero - context:_glContext]; + _glkView = [[GLKView alloc] initWithFrame:CGRectZero context:_glContext]; _glkView.drawableColorFormat = GLKViewDrawableColorFormatRGBA8888; _glkView.drawableDepthFormat = GLKViewDrawableDepthFormatNone; _glkView.drawableStencilFormat = GLKViewDrawableStencilFormatNone; @@ -115,8 +113,7 @@ - (BOOL)configure { // Listen to application state in order to clean up OpenGL before app goes // away. - NSNotificationCenter *notificationCenter = - [NSNotificationCenter defaultCenter]; + NSNotificationCenter *notificationCenter = [NSNotificationCenter defaultCenter]; [notificationCenter addObserver:self selector:@selector(willResignActive) name:UIApplicationWillResignActiveNotification @@ -130,7 +127,7 @@ - (BOOL)configure { // using a refresh rate proportional to screen refresh frequency. This // occurs on the main thread. __weak RTC_OBJC_TYPE(RTCEAGLVideoView) *weakSelf = self; - _timer = [[RTCDisplayLinkTimer alloc] initWithTimerHandler:^{ + _timer = [[RTC_OBJC_TYPE(RTCDisplayLinkTimer) alloc] initWithTimerHandler:^{ RTC_OBJC_TYPE(RTCEAGLVideoView) *strongSelf = weakSelf; [strongSelf displayLinkTimerDidFire]; }]; @@ -141,14 +138,13 @@ - (BOOL)configure { } - (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { - [super setMultipleTouchEnabled:multipleTouchEnabled]; - _glkView.multipleTouchEnabled = multipleTouchEnabled; + [super setMultipleTouchEnabled:multipleTouchEnabled]; + _glkView.multipleTouchEnabled = multipleTouchEnabled; } - (void)dealloc { [[NSNotificationCenter defaultCenter] removeObserver:self]; - UIApplicationState appState = - [UIApplication sharedApplication].applicationState; + UIApplicationState appState = [UIApplication sharedApplication].applicationState; if (appState == UIApplicationStateActive) { [self teardownGL]; } @@ -189,14 +185,14 @@ - (void)glkView:(GLKView *)view drawInRect:(CGRect)rect { return; } RTCVideoRotation rotation = frame.rotation; - if(_rotationOverride != nil) { - [_rotationOverride getValue: &rotation]; + if (_rotationOverride != nil) { + [_rotationOverride getValue:&rotation]; } [self ensureGLContext]; glClear(GL_COLOR_BUFFER_BIT); if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { if (!_nv12TextureCache) { - _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext]; + _nv12TextureCache = [[RTC_OBJC_TYPE(RTCNV12TextureCache) alloc] initWithContext:_glContext]; } if (_nv12TextureCache) { [_nv12TextureCache uploadFrameToTextures:frame]; @@ -211,7 +207,7 @@ - (void)glkView:(GLKView *)view drawInRect:(CGRect)rect { } } else { if (!_i420TextureCache) { - _i420TextureCache = [[RTCI420TextureCache alloc] initWithContext:_glContext]; + _i420TextureCache = [[RTC_OBJC_TYPE(RTCI420TextureCache) alloc] initWithContext:_glContext]; } [_i420TextureCache uploadFrameToTextures:frame]; [_shader applyShadingForFrameWithWidth:frame.width diff --git a/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h b/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h index 9fdcc5a695..2c2319d043 100644 --- a/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h +++ b/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h @@ -11,7 +11,7 @@ #import "RTCOpenGLDefines.h" #import "base/RTCVideoFrame.h" -@interface RTCI420TextureCache : NSObject +@interface RTC_OBJC_TYPE(RTCI420TextureCache) : NSObject @property(nonatomic, readonly) GLuint yTexture; @property(nonatomic, readonly) GLuint uTexture; diff --git a/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm b/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm index a91e927cb4..1e1c2bd189 100644 --- a/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm +++ b/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm @@ -24,7 +24,7 @@ static const GLsizei kNumTexturesPerSet = 3; static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets; -@implementation RTCI420TextureCache { +@implementation RTC_OBJC_TYPE(RTCI420TextureCache) { BOOL _hasUnpackRowLength; GLint _currentTextureSet; // Handles for OpenGL constructs. diff --git a/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.m b/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.m new file mode 100644 index 0000000000..97957faf24 --- /dev/null +++ b/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.m @@ -0,0 +1,199 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#if !TARGET_OS_IPHONE + +#import "RTCNSGLVideoView.h" + +#import +#import +#import + +#import "RTCDefaultShader.h" +#import "RTCI420TextureCache.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" + +@interface RTC_OBJC_TYPE (RTCNSGLVideoView) +() + // `videoFrame` is set when we receive a frame from a worker thread and is read + // from the display link callback so atomicity is required. + @property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * + videoFrame; +@property(atomic, strong) RTC_OBJC_TYPE(RTCI420TextureCache) *i420TextureCache; + +- (void)drawFrame; +@end + +static CVReturn OnDisplayLinkFired(CVDisplayLinkRef displayLink, + const CVTimeStamp *now, + const CVTimeStamp *outputTime, + CVOptionFlags flagsIn, + CVOptionFlags *flagsOut, + void *displayLinkContext) { + RTC_OBJC_TYPE(RTCNSGLVideoView) *view = + (__bridge RTC_OBJC_TYPE(RTCNSGLVideoView) *)displayLinkContext; + [view drawFrame]; + return kCVReturnSuccess; +} + +@implementation RTC_OBJC_TYPE (RTCNSGLVideoView) { + CVDisplayLinkRef _displayLink; + RTC_OBJC_TYPE(RTCVideoFrame) * _lastDrawnFrame; + id _shader; +} + +@synthesize delegate = _delegate; +@synthesize videoFrame = _videoFrame; +@synthesize i420TextureCache = _i420TextureCache; + +- (instancetype)initWithFrame:(NSRect)frame pixelFormat:(NSOpenGLPixelFormat *)format { + return [self initWithFrame:frame pixelFormat:format shader:[[RTC_OBJC_TYPE(RTCDefaultShader) alloc] init]]; +} + +- (instancetype)initWithFrame:(NSRect)frame + pixelFormat:(NSOpenGLPixelFormat *)format + shader:(id)shader { + if (self = [super initWithFrame:frame pixelFormat:format]) { + _shader = shader; + } + return self; +} + +- (void)dealloc { + [self teardownDisplayLink]; +} + +- (void)drawRect:(NSRect)rect { + [self drawFrame]; +} + +- (void)reshape { + [super reshape]; + NSRect frame = [self frame]; + [self ensureGLContext]; + CGLLockContext([[self openGLContext] CGLContextObj]); + glViewport(0, 0, frame.size.width, frame.size.height); + CGLUnlockContext([[self openGLContext] CGLContextObj]); +} + +- (void)lockFocus { + NSOpenGLContext *context = [self openGLContext]; + [super lockFocus]; + if ([context view] != self) { + [context setView:self]; + } + [context makeCurrentContext]; +} + +- (void)prepareOpenGL { + [super prepareOpenGL]; + [self ensureGLContext]; + glDisable(GL_DITHER); + [self setupDisplayLink]; +} + +- (void)clearGLContext { + [self ensureGLContext]; + self.i420TextureCache = nil; + [super clearGLContext]; +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +// These methods may be called on non-main thread. +- (void)setSize:(CGSize)size { + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate videoView:self didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + self.videoFrame = frame; +} + +#pragma mark - Private + +- (void)drawFrame { + RTC_OBJC_TYPE(RTCVideoFrame) *frame = self.videoFrame; + if (!frame || frame == _lastDrawnFrame) { + return; + } + // This method may be called from CVDisplayLink callback which isn't on the + // main thread so we have to lock the GL context before drawing. + NSOpenGLContext *context = [self openGLContext]; + CGLLockContext([context CGLContextObj]); + + [self ensureGLContext]; + glClear(GL_COLOR_BUFFER_BIT); + + // Rendering native CVPixelBuffer is not supported on OS X. + // TODO(magjed): Add support for NV12 texture cache on OS X. + frame = [frame newI420VideoFrame]; + if (!self.i420TextureCache) { + self.i420TextureCache = [[RTC_OBJC_TYPE(RTCI420TextureCache) alloc] initWithContext:context]; + } + RTC_OBJC_TYPE(RTCI420TextureCache) *i420TextureCache = self.i420TextureCache; + if (i420TextureCache) { + [i420TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithWidth:frame.width + height:frame.height + rotation:frame.rotation + yPlane:i420TextureCache.yTexture + uPlane:i420TextureCache.uTexture + vPlane:i420TextureCache.vTexture]; + [context flushBuffer]; + _lastDrawnFrame = frame; + } + CGLUnlockContext([context CGLContextObj]); +} + +- (void)setupDisplayLink { + if (_displayLink) { + return; + } + // Synchronize buffer swaps with vertical refresh rate. + GLint swapInt = 1; + [[self openGLContext] setValues:&swapInt forParameter:NSOpenGLCPSwapInterval]; + + // Create display link. + CVDisplayLinkCreateWithActiveCGDisplays(&_displayLink); + CVDisplayLinkSetOutputCallback(_displayLink, + &OnDisplayLinkFired, + (__bridge void *)self); + // Set the display link for the current renderer. + CGLContextObj cglContext = [[self openGLContext] CGLContextObj]; + CGLPixelFormatObj cglPixelFormat = [[self pixelFormat] CGLPixelFormatObj]; + CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext( + _displayLink, cglContext, cglPixelFormat); + CVDisplayLinkStart(_displayLink); +} + +- (void)teardownDisplayLink { + if (!_displayLink) { + return; + } + CVDisplayLinkRelease(_displayLink); + _displayLink = NULL; +} + +- (void)ensureGLContext { + NSOpenGLContext* context = [self openGLContext]; + NSAssert(context, @"context shouldn't be nil"); + if ([NSOpenGLContext currentContext] != context) { + [context makeCurrentContext]; + } +} + +@end + +#endif // !TARGET_OS_IPHONE diff --git a/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h b/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h index f202b836b5..420490b1ab 100644 --- a/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h +++ b/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h @@ -16,7 +16,7 @@ NS_ASSUME_NONNULL_BEGIN -@interface RTCNV12TextureCache : NSObject +@interface RTC_OBJC_TYPE(RTCNV12TextureCache) : NSObject @property(nonatomic, readonly) GLuint yTexture; @property(nonatomic, readonly) GLuint uvTexture; diff --git a/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m b/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m index a520ac45b4..096767be55 100644 --- a/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m +++ b/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m @@ -14,7 +14,7 @@ #import "base/RTCVideoFrameBuffer.h" #import "components/video_frame_buffer/RTCCVPixelBuffer.h" -@implementation RTCNV12TextureCache { +@implementation RTC_OBJC_TYPE(RTCNV12TextureCache) { CVOpenGLESTextureCacheRef _textureCache; CVOpenGLESTextureRef _yTextureRef; CVOpenGLESTextureRef _uvTextureRef; diff --git a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m index 8de55bde4a..3cc92382e6 100644 --- a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m +++ b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m @@ -55,11 +55,13 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) if ([RTC_OBJC_TYPE(RTCVideoEncoderVP9) isSupported]) { [result - addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp9Name]]; + addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp9Name parameters:nil scalabilityModes:[RTC_OBJC_TYPE(RTCVideoEncoderVP9) scalabilityModes]]]; } #if defined(RTC_USE_LIBAOM_AV1_ENCODER) - [result addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecAv1Name]]; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *av1Info = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecAv1Name parameters:nil scalabilityModes:[RTC_OBJC_TYPE(RTCVideoEncoderAV1) scalabilityModes]]; + [result addObject:av1Info]; #endif return result; diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h index 8ba7584c8e..4070af22e4 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h +++ b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h @@ -13,4 +13,4 @@ RTC_OBJC_EXPORT @end -NS_ASSUME_NONNULL_END \ No newline at end of file +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm index 6ba2074768..2af8a63500 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm +++ b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm @@ -4,6 +4,15 @@ #import "RTCVideoCodecInfo.h" #import "RTCVideoEncoderFactorySimulcast.h" #import "api/video_codec/RTCVideoEncoderSimulcast.h" +#import "api/peerconnection/RTCVideoCodecInfo+Private.h" + +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "media/base/media_constants.h" @interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) () @@ -28,12 +37,27 @@ - (instancetype)initWithPrimary:(id)prima } - (nullable id)createEncoder: (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { - return [RTCVideoEncoderSimulcast simulcastEncoderWithPrimary: _primary fallback: _fallback videoCodecInfo: info]; + return [RTC_OBJC_TYPE(RTCVideoEncoderSimulcast) simulcastEncoderWithPrimary: _primary fallback: _fallback videoCodecInfo: info]; } - (NSArray *)supportedCodecs { - return [[_primary supportedCodecs] arrayByAddingObjectsFromArray: [_fallback supportedCodecs]]; + NSArray *supportedCodecs = [[_primary supportedCodecs] arrayByAddingObjectsFromArray: [_fallback supportedCodecs]]; + + NSMutableArray *addingCodecs = [[NSMutableArray alloc] init]; + + for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs(true)) { + RTCVideoCodecInfo *codec = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat: format]; + [addingCodecs addObject: codec]; + } + + auto av1Format = webrtc::SdpVideoFormat( + cricket::kAv1CodecName, webrtc::CodecParameterMap(), + webrtc::LibaomAv1EncoderSupportedScalabilityModes()); + RTCVideoCodecInfo *av1Codec = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat: av1Format]; + [addingCodecs addObject: av1Codec]; + + return [supportedCodecs arrayByAddingObjectsFromArray: addingCodecs]; } -@end \ No newline at end of file +@end diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm index 2160d79ae5..d3dd33aef6 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm +++ b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm @@ -54,14 +54,42 @@ - (void)frameWasEncoded : (OSStatus)status flags : (VTEncodeInfoFlags)infoFlags // The ratio between kVTCompressionPropertyKey_DataRateLimits and // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher // than the average bit rate to avoid undershooting the target. -const float kLimitToAverageBitRateFactor = 1.5f; +const float kLimitToAverageBitRateFactor = 10.0f; // These thresholds deviate from the default h264 QP thresholds, as they // have been found to work better on devices that support VideoToolbox const int kLowH264QpThreshold = 28; const int kHighH264QpThreshold = 39; +const int kBitsPerByte = 8; const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; +typedef NS_ENUM(NSInteger, RTCVideoEncodeMode) { + Variable = 0, + Constant = 1, +}; + +NSArray *CreateRateLimitArray(uint32_t computedBitrateBps, RTCVideoEncodeMode mode) { + switch (mode) { + case Variable: { + // 5 seconds should be an okay interval for VBR to enforce the long-term + // limit. + float avgInterval = 5.0; + uint32_t avgBytesPerSecond = computedBitrateBps / kBitsPerByte * avgInterval; + // And the peak bitrate is measured per-second in a way similar to CBR. + float peakInterval = 1.0; + uint32_t peakBytesPerSecond = + computedBitrateBps * kLimitToAverageBitRateFactor / kBitsPerByte; + return @[ @(peakBytesPerSecond), @(peakInterval), @(avgBytesPerSecond), @(avgInterval) ]; + } + case Constant: { + // CBR should be enforces with granularity of a second. + float targetInterval = 1.0; + int32_t targetBitrate = computedBitrateBps / kBitsPerByte; + return @[ @(targetBitrate), @(targetInterval) ]; + } + } +} + // Struct that we pass to the encoder per frame to encode. We receive it again // in the encoder callback. struct RTCFrameEncodeParams { @@ -180,10 +208,13 @@ void compressionOutputCallback(void *encoder, // no specific VideoToolbox profile for the specified level, AutoLevel will be // returned. The user must initialize the encoder with a resolution and // framerate conforming to the selected H264 level regardless. -CFStringRef ExtractProfile(const webrtc::H264ProfileLevelId &profile_level_id) { +CFStringRef ExtractProfile(const webrtc::H264ProfileLevelId &profile_level_id, bool screenSharing) { switch (profile_level_id.profile) { case webrtc::H264Profile::kProfileConstrainedBaseline: case webrtc::H264Profile::kProfileBaseline: + if (screenSharing) { + return kVTProfileLevel_H264_Baseline_AutoLevel; + } switch (profile_level_id.level) { case webrtc::H264Level::kLevel3: return kVTProfileLevel_H264_Baseline_3_0; @@ -319,8 +350,8 @@ NSUInteger GetMaxSampleRate(const webrtc::H264ProfileLevelId &profile_level_id) @implementation RTC_OBJC_TYPE (RTCVideoEncoderH264) { RTC_OBJC_TYPE(RTCVideoCodecInfo) * _codecInfo; - std::unique_ptr _bitrateAdjuster; uint32_t _targetBitrateBps; + uint32_t _targetFrameRate; uint32_t _encoderBitrateBps; uint32_t _encoderFrameRate; uint32_t _maxAllowedFrameRate; @@ -330,10 +361,17 @@ @implementation RTC_OBJC_TYPE (RTCVideoEncoderH264) { int32_t _width; int32_t _height; VTCompressionSessionRef _compressionSession; - RTCVideoCodecMode _mode; + CVPixelBufferPoolRef _pixelBufferPool; + RTCVideoCodecMode _codecMode; + unsigned int _maxQP; + unsigned int _minBitrate; + unsigned int _maxBitrate; + RTCVideoEncodeMode _encodeMode; webrtc::H264BitstreamParser _h264BitstreamParser; std::vector _frameScaleBuffer; + + CMTime _previousPresentationTimeStamp; } // .5 is set as a mininum to prevent overcompensating for large temporary @@ -346,12 +384,14 @@ @implementation RTC_OBJC_TYPE (RTCVideoEncoderH264) { - (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo { if (self = [super init]) { _codecInfo = codecInfo; - _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95)); _packetizationMode = RTCH264PacketizationModeNonInterleaved; _profile_level_id = webrtc::ParseSdpForH264ProfileLevelId([codecInfo nativeSdpVideoFormat].parameters); + _previousPresentationTimeStamp = kCMTimeZero; RTC_DCHECK(_profile_level_id); - RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(ExtractProfile(*_profile_level_id)); + RTC_LOG(LS_INFO) << "Using profile " + << CFStringToString(ExtractProfile( + *_profile_level_id, _codecMode == RTCVideoCodecModeScreensharing)); RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]); } return self; @@ -368,7 +408,12 @@ - (NSInteger)startEncodeWithSettings:(RTC_OBJC_TYPE(RTCVideoEncoderSettings) *)s _width = settings.width; _height = settings.height; - _mode = settings.mode; + _codecMode = settings.mode; + _maxQP = settings.qpMax; + + _encodeMode = Variable; // Always variable mode for now + _minBitrate = settings.minBitrate * 1000; // minBitrate is in kbps. + _maxBitrate = settings.maxBitrate * 1000; // maxBitrate is in kbps. uint32_t aligned_width = (((_width + 15) >> 4) << 4); uint32_t aligned_height = (((_height + 15) >> 4) << 4); @@ -376,9 +421,15 @@ - (NSInteger)startEncodeWithSettings:(RTC_OBJC_TYPE(RTCVideoEncoderSettings) *)s (aligned_width * aligned_height)); // We can only set average bitrate on the HW encoder. - _targetBitrateBps = settings.startBitrate * 1000; // startBitrate is in kbps. - _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); - _encoderFrameRate = MIN(settings.maxFramerate, _maxAllowedFrameRate); + if (_encodeMode == Constant) { + _targetBitrateBps = _maxBitrate; + } else { + _targetBitrateBps = settings.startBitrate * 1000; // startBitrate is in kbps. + } + + _targetFrameRate = MIN(settings.maxFramerate, _maxAllowedFrameRate); + _encoderBitrateBps = 0; + _encoderFrameRate = 0; if (settings.maxFramerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) { RTC_LOG(LS_WARNING) << "Initial encoder frame rate setting " << settings.maxFramerate << " is larger than the " @@ -397,8 +448,15 @@ - (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame if (!_callback || !_compressionSession) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - BOOL isKeyframeRequired = NO; + CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000); + if (CMTimeCompare(presentationTimeStamp, _previousPresentationTimeStamp) == 0) { + // Same PTS + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + _previousPresentationTimeStamp = presentationTimeStamp; + + BOOL isKeyframeRequired = NO; // Get a pixel buffer from the pool and copy frame data over. if ([self resetCompressionSessionIfNeededWithFrame:frame]) { isKeyframeRequired = YES; @@ -425,8 +483,8 @@ - (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame int dstWidth = CVPixelBufferGetWidth(pixelBuffer); int dstHeight = CVPixelBufferGetHeight(pixelBuffer); if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) { - int size = - [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth height:dstHeight]; + int size = [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth + height:dstHeight]; _frameScaleBuffer.resize(size); } else { _frameScaleBuffer.clear(); @@ -466,7 +524,6 @@ - (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame } } - CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000); CFDictionaryRef frameProperties = nullptr; if (isKeyframeRequired) { CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; @@ -484,8 +541,8 @@ - (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame frame.rotation)); encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode; - // Update the bitrate if needed. - [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:_encoderFrameRate]; + // Update encoder bitrate or frameRate if needed. + [self updateEncoderBitrateAndFrameRate]; OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession, pixelBuffer, @@ -526,14 +583,19 @@ - (void)setCallback:(RTCVideoEncoderCallback)callback { } - (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { - _targetBitrateBps = 1000 * bitrateKbit; - _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); + // set target bitrate bps + _targetBitrateBps = bitrateKbit * 1000; + + RTC_LOG(LS_INFO) << "setBitrateKBit: " << bitrateKbit << " targetBps: " << _targetBitrateBps + << " frameRate: " << framerate; + if (framerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) { RTC_LOG(LS_WARNING) << "Encoder frame rate setting " << framerate << " is larger than the " << "maximal allowed frame rate " << _maxAllowedFrameRate << "."; } - framerate = MIN(framerate, _maxAllowedFrameRate); - [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:framerate]; + + _targetFrameRate = MIN(framerate, _maxAllowedFrameRate); + return WEBRTC_VIDEO_CODEC_OK; } @@ -585,7 +647,8 @@ - (BOOL)resetCompressionSessionIfNeededWithFrame:(RTC_OBJC_TYPE(RTCVideoFrame) * CVPixelBufferPoolRef pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession); if (!pixelBufferPool) { - return NO; + [self resetCompressionSessionWithPixelFormat:framePixelFormat]; + return YES; } NSDictionary *poolAttributes = @@ -631,14 +694,19 @@ - (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat { (NSString *)kCVPixelBufferPixelFormatTypeKey : @(framePixelFormat), }; - NSDictionary *encoder_specs; + NSMutableDictionary *encoder_specs; #if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) // Currently hw accl is supported above 360p on mac, below 360p // the compression session will be created with hw accl disabled. - encoder_specs = @{ + encoder_specs = [@{ (NSString *)kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder : @(YES), - }; - + } mutableCopy]; + // Enable low-latency video encoding + if (@available(iOS 14.5, macOS 11.3, *)) { + [encoder_specs addEntriesFromDictionary:@{ + (NSString *)kVTVideoEncoderSpecification_EnableLowLatencyRateControl : @(YES), + }]; + } #endif OSStatus status = VTCompressionSessionCreate( nullptr, // use default allocator @@ -675,11 +743,30 @@ - (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat { - (void)configureCompressionSession { RTC_DCHECK(_compressionSession); SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true); - SetVTSessionProperty(_compressionSession, - kVTCompressionPropertyKey_ProfileLevel, - ExtractProfile(*_profile_level_id)); + // Sacrifice encoding speed over quality when necessary + if (@available(iOS 14.0, macOS 11.0, *)) { + SetVTSessionProperty( + _compressionSession, kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, true); + } + // Set maximum QP for screen sharing mode, range must be within 1 to 51 + // https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_maxallowedframeqp + if (@available(iOS 15.0, macOS 12.0, *)) { + // Only enable for screen sharing and let VideoToolbox do the optimizing as much as possible. + if (_codecMode == RTCVideoCodecModeScreensharing) { + RTC_LOG(LS_INFO) << "Configuring VideoToolbox to use maxQP: " << kHighH264QpThreshold + << " mode: " << _codecMode; + SetVTSessionProperty( + _compressionSession, kVTCompressionPropertyKey_MaxAllowedFrameQP, kHighH264QpThreshold); + } + } + SetVTSessionProperty( + _compressionSession, + kVTCompressionPropertyKey_ProfileLevel, + ExtractProfile(*_profile_level_id, _codecMode == RTCVideoCodecModeScreensharing)); SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false); - [self setEncoderBitrateBps:_targetBitrateBps frameRate:_encoderFrameRate]; + + // [self updateEncoderBitrateAndFrameRate]; + // TODO(tkchin): Look at entropy mode and colorspace matrices. // TODO(tkchin): Investigate to see if there's any way to make this work. // May need it to interop with Android. Currently this call just fails. @@ -706,49 +793,59 @@ - (NSString *)implementationName { return @"VideoToolbox"; } -- (void)setBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate { - if (_encoderBitrateBps != bitrateBps || _encoderFrameRate != frameRate) { - [self setEncoderBitrateBps:bitrateBps frameRate:frameRate]; +- (void)updateEncoderBitrateAndFrameRate { + // If no compression session simply return + if (!_compressionSession) { + return; } -} + // Initial status + OSStatus status = noErr; -- (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate { - if (_compressionSession) { - SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps); + uint32_t computedBitrateBps = _targetBitrateBps; - // With zero `_maxAllowedFrameRate`, we fall back to automatic frame rate detection. - if (_maxAllowedFrameRate > 0) { - SetVTSessionProperty( - _compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, frameRate); - } + // With zero `_maxAllowedFrameRate`, we fall back to automatic frame rate detection. + uint32_t computedFrameRate = _maxAllowedFrameRate > 0 ? _targetFrameRate : 0; - // TODO(tkchin): Add a helper method to set array value. - int64_t dataLimitBytesPerSecondValue = - static_cast(bitrateBps * kLimitToAverageBitRateFactor / 8); - CFNumberRef bytesPerSecond = - CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue); - int64_t oneSecondValue = 1; - CFNumberRef oneSecond = - CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue); - const void *nums[2] = {bytesPerSecond, oneSecond}; - CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); - OSStatus status = VTSessionSetProperty( - _compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits); - if (bytesPerSecond) { - CFRelease(bytesPerSecond); - } - if (oneSecond) { - CFRelease(oneSecond); + // Set frame rate + if (computedFrameRate != _encoderFrameRate) { + status = VTSessionSetProperty(_compressionSession, + kVTCompressionPropertyKey_ExpectedFrameRate, + (__bridge CFTypeRef) @(computedFrameRate)); + // Ensure the bitrate was set successfully + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to set frame rate: " << computedFrameRate + << " error: " << status; + } else { + RTC_LOG(LS_INFO) << "Did update encoder frame rate: " << computedFrameRate; } - if (dataRateLimits) { - CFRelease(dataRateLimits); + _encoderFrameRate = computedFrameRate; + } + + // Set bitrate + if (computedBitrateBps != _encoderBitrateBps) { + status = VTSessionSetProperty(_compressionSession, + kVTCompressionPropertyKey_AverageBitRate, + (__bridge CFTypeRef) @(computedBitrateBps)); + + // Ensure the bitrate was set successfully + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to update encoder bitrate: " << computedBitrateBps + << "error: " << status; + } else { + RTC_LOG(LS_INFO) << "Did update encoder bitrate: " << computedBitrateBps; } + + status = VTSessionSetProperty( + _compressionSession, + kVTCompressionPropertyKey_DataRateLimits, + (__bridge CFArrayRef)CreateRateLimitArray(computedBitrateBps, _encodeMode)); if (status != noErr) { - RTC_LOG(LS_ERROR) << "Failed to set data rate limit with code: " << status; + RTC_LOG(LS_ERROR) << "Failed to update encoder data rate limits"; + } else { + RTC_LOG(LS_INFO) << "Did update encoder data rate limits"; } - _encoderBitrateBps = bitrateBps; - _encoderFrameRate = frameRate; + _encoderBitrateBps = computedBitrateBps; } } @@ -804,8 +901,9 @@ - (void)frameWasEncoded:(OSStatus)status frame.captureTimeMs = renderTimeMs; frame.timeStamp = timestamp; frame.rotation = rotation; - frame.contentType = (_mode == RTCVideoCodecModeScreensharing) ? RTCVideoContentTypeScreenshare : - RTCVideoContentTypeUnspecified; + frame.contentType = (_codecMode == RTCVideoCodecModeScreensharing) ? + RTCVideoContentTypeScreenshare : + RTCVideoContentTypeUnspecified; frame.flags = webrtc::VideoSendTiming::kInvalid; _h264BitstreamParser.ParseBitstream(*buffer); @@ -816,7 +914,6 @@ - (void)frameWasEncoded:(OSStatus)status RTC_LOG(LS_ERROR) << "Encode callback failed"; return; } - _bitrateAdjuster->Update(frame.buffer.length); } - (nullable RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *)scalingSettings { @@ -826,3 +923,4 @@ - (void)frameWasEncoded:(OSStatus)status } @end + diff --git a/sdk/objc/helpers/RTCYUVHelper.h b/sdk/objc/helpers/RTCYUVHelper.h new file mode 100644 index 0000000000..ec8ce48355 --- /dev/null +++ b/sdk/objc/helpers/RTCYUVHelper.h @@ -0,0 +1,118 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" +#import "RTCVideoFrame.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCYUVHelper) : NSObject + +- (instancetype)init NS_UNAVAILABLE; + ++ (void)I420Rotate:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstY:(uint8_t*)dstY + dstStrideY:(int)dstStrideY + dstU:(uint8_t*)dstU + dstStrideU:(int)dstStrideU + dstV:(uint8_t*)dstV + dstStrideV:(int)dstStrideV + width:(int)width + height:(int)height + mode:(RTCVideoRotation)mode; + ++ (int)I420ToNV12:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstY:(uint8_t*)dstY + dstStrideY:(int)dstStrideY + dstUV:(uint8_t*)dstUV + dstStrideUV:(int)dstStrideUV + width:(int)width + height:(int)height; + ++ (int)I420ToNV21:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstY:(uint8_t*)dstY + dstStrideY:(int)dstStrideY + dstUV:(uint8_t*)dstUV + dstStrideUV:(int)dstStrideUV + width:(int)width + height:(int)height; + ++ (int)I420ToARGB:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstARGB:(uint8_t*)dstARGB + dstStrideARGB:(int)dstStrideARGB + width:(int)width + height:(int)height; + ++ (int)I420ToBGRA:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstBGRA:(uint8_t*)dstBGRA + dstStrideBGRA:(int)dstStrideBGRA + width:(int)width + height:(int)height; + ++ (int)I420ToABGR:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstABGR:(uint8_t*)dstABGR + dstStrideABGR:(int)dstStrideABGR + width:(int)width + height:(int)height; + ++ (int)I420ToRGBA:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstRGBA:(uint8_t*)dstRGBA + dstStrideRGBA:(int)dstStrideRGBA + width:(int)width + height:(int)height; + ++ (int)I420ToRGB24:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstRGB24:(uint8_t*)dstRGB24 + dstStrideRGB24:(int)dstStrideRGB24 + width:(int)width + height:(int)height; + +@end diff --git a/sdk/objc/helpers/RTCYUVHelper.mm b/sdk/objc/helpers/RTCYUVHelper.mm new file mode 100644 index 0000000000..4a39d469da --- /dev/null +++ b/sdk/objc/helpers/RTCYUVHelper.mm @@ -0,0 +1,179 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCYUVHelper.h" + +#include "third_party/libyuv/include/libyuv.h" + +@implementation RTC_OBJC_TYPE (RTCYUVHelper) + ++ (void)I420Rotate:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstY:(uint8_t*)dstY + dstStrideY:(int)dstStrideY + dstU:(uint8_t*)dstU + dstStrideU:(int)dstStrideU + dstV:(uint8_t*)dstV + dstStrideV:(int)dstStrideV + width:(int)width + height:(int)height + mode:(RTCVideoRotation)mode { + libyuv::I420Rotate(srcY, + srcStrideY, + srcU, + srcStrideU, + srcV, + srcStrideV, + dstY, + dstStrideY, + dstU, + dstStrideU, + dstV, + dstStrideV, + width, + height, + (libyuv::RotationMode)mode); +} + ++ (int)I420ToNV12:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstY:(uint8_t*)dstY + dstStrideY:(int)dstStrideY + dstUV:(uint8_t*)dstUV + dstStrideUV:(int)dstStrideUV + width:(int)width + height:(int)height { + return libyuv::I420ToNV12(srcY, + srcStrideY, + srcU, + srcStrideU, + srcV, + srcStrideV, + dstY, + dstStrideY, + dstUV, + dstStrideUV, + width, + height); +} + ++ (int)I420ToNV21:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstY:(uint8_t*)dstY + dstStrideY:(int)dstStrideY + dstUV:(uint8_t*)dstUV + dstStrideUV:(int)dstStrideUV + width:(int)width + height:(int)height { + return libyuv::I420ToNV21(srcY, + srcStrideY, + srcU, + srcStrideU, + srcV, + srcStrideV, + dstY, + dstStrideY, + dstUV, + dstStrideUV, + width, + height); +} + ++ (int)I420ToARGB:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstARGB:(uint8_t*)dstARGB + dstStrideARGB:(int)dstStrideARGB + width:(int)width + height:(int)height { + return libyuv::I420ToARGB( + srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstARGB, dstStrideARGB, width, height); +} + ++ (int)I420ToBGRA:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstBGRA:(uint8_t*)dstBGRA + dstStrideBGRA:(int)dstStrideBGRA + width:(int)width + height:(int)height { + return libyuv::I420ToBGRA( + srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstBGRA, dstStrideBGRA, width, height); +} + ++ (int)I420ToABGR:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstABGR:(uint8_t*)dstABGR + dstStrideABGR:(int)dstStrideABGR + width:(int)width + height:(int)height { + return libyuv::I420ToABGR( + srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstABGR, dstStrideABGR, width, height); +} + ++ (int)I420ToRGBA:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstRGBA:(uint8_t*)dstRGBA + dstStrideRGBA:(int)dstStrideRGBA + width:(int)width + height:(int)height { + return libyuv::I420ToRGBA( + srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstRGBA, dstStrideRGBA, width, height); +} + ++ (int)I420ToRGB24:(const uint8_t*)srcY + srcStrideY:(int)srcStrideY + srcU:(const uint8_t*)srcU + srcStrideU:(int)srcStrideU + srcV:(const uint8_t*)srcV + srcStrideV:(int)srcStrideV + dstRGB24:(uint8_t*)dstRGB24 + dstStrideRGB24:(int)dstStrideRGB24 + width:(int)width + height:(int)height { + return libyuv::I420ToRGB24(srcY, + srcStrideY, + srcU, + srcStrideU, + srcV, + srcStrideV, + dstRGB24, + dstStrideRGB24, + width, + height); +} + +@end diff --git a/sdk/objc/native/api/video_capturer.mm b/sdk/objc/native/api/video_capturer.mm index a7260ab802..d5cf6fd563 100644 --- a/sdk/objc/native/api/video_capturer.mm +++ b/sdk/objc/native/api/video_capturer.mm @@ -20,7 +20,7 @@ RTC_OBJC_TYPE(RTCVideoCapturer) * objc_video_capturer, rtc::Thread *signaling_thread, rtc::Thread *worker_thread) { - RTCObjCVideoSourceAdapter *adapter = [[RTCObjCVideoSourceAdapter alloc] init]; + RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter) *adapter = [[RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter) alloc] init]; rtc::scoped_refptr objc_video_track_source = rtc::make_ref_counted(adapter); rtc::scoped_refptr video_source = diff --git a/sdk/objc/native/src/audio/audio_device_ios.h b/sdk/objc/native/src/audio/audio_device_ios.h index 877cb8bbf9..506487a1c2 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.h +++ b/sdk/objc/native/src/audio/audio_device_ios.h @@ -25,7 +25,7 @@ #include "sdk/objc/base/RTCMacros.h" #include "voice_processing_audio_unit.h" -RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter); +RTC_FWD_DECL_OBJC_CLASS(RTC_OBJC_TYPE(RTCNativeAudioSessionDelegateAdapter)); namespace webrtc { @@ -279,7 +279,7 @@ class AudioDeviceIOS : public AudioDeviceGeneric, bool is_interrupted_; // Audio interruption observer instance. - RTCNativeAudioSessionDelegateAdapter* audio_session_observer_ + RTC_OBJC_TYPE(RTCNativeAudioSessionDelegateAdapter)* audio_session_observer_ RTC_GUARDED_BY(thread_); // Set to true if we've activated the audio session. diff --git a/sdk/objc/native/src/audio/audio_device_ios.mm b/sdk/objc/native/src/audio/audio_device_ios.mm index ed7260489a..cd5f32aa72 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_ios.mm @@ -121,7 +121,7 @@ static void LogDeviceInfo() { io_thread_checker_.Detach(); thread_ = rtc::Thread::Current(); - audio_session_observer_ = [[RTCNativeAudioSessionDelegateAdapter alloc] initWithObserver:this]; + audio_session_observer_ = [[RTC_OBJC_TYPE(RTCNativeAudioSessionDelegateAdapter) alloc] initWithObserver:this]; mach_timebase_info_data_t tinfo; mach_timebase_info(&tinfo); machTickUnitsToNanoseconds_ = (double)tinfo.numer / tinfo.denom; diff --git a/sdk/objc/native/src/objc_audio_device.h b/sdk/objc/native/src/objc_audio_device.h index fcfe7a6e8b..88f6f19f99 100644 --- a/sdk/objc/native/src/objc_audio_device.h +++ b/sdk/objc/native/src/objc_audio_device.h @@ -19,7 +19,7 @@ #include "modules/audio_device/include/audio_device.h" #include "rtc_base/thread.h" -@class ObjCAudioDeviceDelegate; +@class RTC_OBJC_TYPE(ObjCAudioDeviceDelegate); namespace webrtc { @@ -267,7 +267,7 @@ class ObjCAudioDeviceModule : public AudioDeviceModule { rtc::BufferT record_audio_buffer_; // Delegate object provided to RTCAudioDevice during initialization - ObjCAudioDeviceDelegate* audio_device_delegate_; + RTC_OBJC_TYPE(ObjCAudioDeviceDelegate)* audio_device_delegate_; }; } // namespace objc_adm diff --git a/sdk/objc/native/src/objc_audio_device.mm b/sdk/objc/native/src/objc_audio_device.mm index d629fae20f..5fb72d8a5c 100644 --- a/sdk/objc/native/src/objc_audio_device.mm +++ b/sdk/objc/native/src/objc_audio_device.mm @@ -77,7 +77,7 @@ if (![audio_device_ isInitialized]) { if (audio_device_delegate_ == nil) { - audio_device_delegate_ = [[ObjCAudioDeviceDelegate alloc] + audio_device_delegate_ = [[RTC_OBJC_TYPE(ObjCAudioDeviceDelegate) alloc] initWithAudioDeviceModule:rtc::scoped_refptr(this) audioDeviceThread:thread_]; } diff --git a/sdk/objc/native/src/objc_audio_device_delegate.h b/sdk/objc/native/src/objc_audio_device_delegate.h index 3af079dad9..0b546f269c 100644 --- a/sdk/objc/native/src/objc_audio_device_delegate.h +++ b/sdk/objc/native/src/objc_audio_device_delegate.h @@ -22,7 +22,7 @@ class ObjCAudioDeviceModule; } // namespace objc_adm } // namespace webrtc -@interface ObjCAudioDeviceDelegate : NSObject +@interface RTC_OBJC_TYPE(ObjCAudioDeviceDelegate) : NSObject - (instancetype)initWithAudioDeviceModule: (rtc::scoped_refptr)audioDeviceModule diff --git a/sdk/objc/native/src/objc_audio_device_delegate.mm b/sdk/objc/native/src/objc_audio_device_delegate.mm index 156d6326a4..f4c8cfb71a 100644 --- a/sdk/objc/native/src/objc_audio_device_delegate.mm +++ b/sdk/objc/native/src/objc_audio_device_delegate.mm @@ -55,7 +55,7 @@ } // namespace -@implementation ObjCAudioDeviceDelegate { +@implementation RTC_OBJC_TYPE(ObjCAudioDeviceDelegate) { rtc::scoped_refptr impl_; } diff --git a/sdk/objc/native/src/objc_desktop_capture.h b/sdk/objc/native/src/objc_desktop_capture.h new file mode 100644 index 0000000000..a781457220 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_capture.h @@ -0,0 +1,70 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_CAPTURE_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_CAPTURE_H_ + +#import "base/RTCMacros.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_and_cursor_composer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "rtc_base/thread.h" + +@protocol RTC_OBJC_TYPE +(DesktopCapturerDelegate); + +namespace webrtc { + +enum DesktopType { kScreen, kWindow }; + +class ObjCDesktopCapturer : public DesktopCapturer::Callback { + public: + enum CaptureState { CS_RUNNING, CS_STOPPED, CS_FAILED}; + + public: + ObjCDesktopCapturer(DesktopType type, + webrtc::DesktopCapturer::SourceId source_id, + id delegate); + virtual ~ObjCDesktopCapturer(); + + virtual CaptureState Start(uint32_t fps); + + virtual void Stop(); + + virtual bool IsRunning(); + + protected: + virtual void OnCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) override; + private: + void CaptureFrame(); + webrtc::DesktopCaptureOptions options_; + std::unique_ptr capturer_; + std::unique_ptr thread_; + CaptureState capture_state_ = CS_STOPPED; + DesktopType type_; + webrtc::DesktopCapturer::SourceId source_id_; + id delegate_; + uint32_t capture_delay_ = 1000; // 1s + webrtc::DesktopCapturer::Result result_ = webrtc::DesktopCapturer::Result::SUCCESS; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_CAPTURE_H_ diff --git a/sdk/objc/native/src/objc_desktop_capture.mm b/sdk/objc/native/src/objc_desktop_capture.mm new file mode 100644 index 0000000000..7aba3e5612 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_capture.mm @@ -0,0 +1,205 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/sequence_checker.h" +#include "rtc_base/checks.h" +#include "sdk/objc/native/src/objc_desktop_capture.h" +#include "sdk/objc/native/src/objc_video_frame.h" +#include "third_party/libyuv/include/libyuv.h" + +#import "components/capturer/RTCDesktopCapturer+Private.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +namespace webrtc { + +enum { kCaptureDelay = 33, kCaptureMessageId = 1000 }; + +ObjCDesktopCapturer::ObjCDesktopCapturer(DesktopType type, + webrtc::DesktopCapturer::SourceId source_id, + id delegate) + : thread_(rtc::Thread::Create()), source_id_(source_id), delegate_(delegate) { + RTC_DCHECK(thread_); + type_ = type; + thread_->Start(); + options_ = webrtc::DesktopCaptureOptions::CreateDefault(); + options_.set_detect_updated_region(true); + options_.set_allow_iosurface(true); + thread_->BlockingCall([this, type] { + if (type == kScreen) { + capturer_ = std::make_unique( + webrtc::DesktopCapturer::CreateScreenCapturer(options_), options_); + } else { + capturer_ = std::make_unique( + webrtc::DesktopCapturer::CreateWindowCapturer(options_), options_); + } + }); +} + +ObjCDesktopCapturer::~ObjCDesktopCapturer() { + thread_->BlockingCall([this] { + capturer_.reset(); + }); +} + +ObjCDesktopCapturer::CaptureState ObjCDesktopCapturer::Start(uint32_t fps) { + if(capture_state_ == CS_RUNNING) { + return capture_state_; + } + + if(fps == 0) { + capture_state_ = CS_FAILED; + return capture_state_; + } + + if (fps >= 60) { + capture_delay_ = uint32_t(1000.0 / 60.0); + } else { + capture_delay_ = uint32_t(1000.0 / fps); + } + + if (source_id_ != -1) { + if (!capturer_->SelectSource(source_id_)) { + capture_state_ = CS_FAILED; + return capture_state_; + } + if (type_ == kWindow) { + if (!capturer_->FocusOnSelectedSource()) { + capture_state_ = CS_FAILED; + return capture_state_; + } + } + } + + thread_->BlockingCall([this] { + capturer_->Start(this); + }); + capture_state_ = CS_RUNNING; + + thread_->PostTask([this] { + CaptureFrame(); + }); + + [delegate_ didSourceCaptureStart]; + return capture_state_; +} + +void ObjCDesktopCapturer::Stop() { + [delegate_ didSourceCaptureStop]; + capture_state_ = CS_STOPPED; +} + +bool ObjCDesktopCapturer::IsRunning() { + return capture_state_ == CS_RUNNING; +} + +void ObjCDesktopCapturer::OnCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) { + if (result != result_) { + if (result == webrtc::DesktopCapturer::Result::ERROR_PERMANENT) { + [delegate_ didSourceCaptureError]; + capture_state_ = CS_FAILED; + return; + } + + if (result == webrtc::DesktopCapturer::Result::ERROR_TEMPORARY) { + result_ = result; + [delegate_ didSourceCapturePaused]; + return; + } + + if (result == webrtc::DesktopCapturer::Result::SUCCESS) { + result_ = result; + [delegate_ didSourceCaptureStart]; + } + } + + if (result == webrtc::DesktopCapturer::Result::ERROR_TEMPORARY) { + return; + } + + int width = frame->size().width(); + int height = frame->size().height(); + int real_width = width; + + if (type_ == kWindow) { + int multiple = 0; +#if defined(WEBRTC_ARCH_X86_FAMILY) + multiple = 16; +#elif defined(WEBRTC_ARCH_ARM64) + multiple = 32; +#endif + // A multiple of $multiple must be used as the width of the src frame, + // and the right black border needs to be cropped during conversion. + if (multiple != 0 && (width % multiple) != 0) { + width = (width / multiple + 1) * multiple; + } + } + + CVPixelBufferRef pixelBuffer = NULL; + + NSDictionary *pixelAttributes = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVReturn res = CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)(pixelAttributes), + &pixelBuffer); + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *pxdata = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer); + libyuv::ConvertToARGB(reinterpret_cast(frame->data()), + real_width * height * 4, + reinterpret_cast(pxdata), + width * 4, + 0, + 0, + width, + height, + real_width, + height, + libyuv::kRotate0, + libyuv::FOURCC_ARGB); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + if (res != kCVReturnSuccess) { + NSLog(@"Unable to create cvpixelbuffer %d", res); + return; + } + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer]; + NSTimeInterval timeStampSeconds = CACurrentMediaTime(); + int64_t timeStampNs = lroundf(timeStampSeconds * NSEC_PER_SEC); + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer + rotation:RTCVideoRotation_0 + timeStampNs:timeStampNs]; + CVPixelBufferRelease(pixelBuffer); + [delegate_ didCaptureVideoFrame:videoFrame]; +} + +void ObjCDesktopCapturer::CaptureFrame() { + RTC_DCHECK_RUN_ON(thread_.get()); + if (capture_state_ == CS_RUNNING) { + capturer_->CaptureFrame(); + thread_->PostDelayedHighPrecisionTask( + [this]() { + CaptureFrame(); + }, + TimeDelta::Millis(capture_delay_)); + } +} + +} // namespace webrtc diff --git a/sdk/objc/native/src/objc_desktop_media_list.h b/sdk/objc/native/src/objc_desktop_media_list.h new file mode 100644 index 0000000000..ecb2d27221 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_media_list.h @@ -0,0 +1,111 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_MEDIA_LIST_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_MEDIA_LIST_H_ + +#import "base/RTCMacros.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "rtc_base/thread.h" + +#include "objc_desktop_capture.h" + +#import "components/capturer/RTCDesktopMediaList+Private.h" + +namespace webrtc { + +class MediaSource { + public: + MediaSource( ObjCDesktopMediaList *mediaList, DesktopCapturer::Source src, DesktopType type) + : source(src), mediaList_(mediaList), type_(type) {} + virtual ~MediaSource() {} + + DesktopCapturer::Source source; + + // source id + DesktopCapturer::SourceId id() const { return source.id; } + + // source name + std::string name() const { return source.title; } + + // Returns the thumbnail of the source, jpeg format. + std::vector thumbnail() const { return thumbnail_; } + + + + DesktopType type() const { return type_; } + + bool UpdateThumbnail(); + + void SaveCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame); + + private: + std::vector thumbnail_; + ObjCDesktopMediaList *mediaList_; + DesktopType type_; +}; + +class ObjCDesktopMediaList { + public: + enum CaptureState { CS_RUNNING, CS_STOPPED, CS_FAILED}; + public: + ObjCDesktopMediaList(DesktopType type, RTC_OBJC_TYPE(RTCDesktopMediaList)* objcMediaList); + + virtual ~ObjCDesktopMediaList(); + + virtual int32_t UpdateSourceList(bool force_reload = false, bool get_thumbnail = true); + + virtual int GetSourceCount() const; + + virtual MediaSource* GetSource(int index); + + virtual bool GetThumbnail(MediaSource *source, bool notify); + + private: + class CallbackProxy : public DesktopCapturer::Callback { + public: + CallbackProxy(){} + void SetCallback(std::function frame)> on_capture_result) { + on_capture_result_ = on_capture_result; + } + private: + void OnCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) override { + if(on_capture_result_) on_capture_result_(result, std::move(frame)); + } + std::function frame)> on_capture_result_ = nullptr; + }; + private: + std::unique_ptr callback_; + webrtc::DesktopCaptureOptions options_; + std::unique_ptr capturer_; + std::unique_ptr thread_; + std::vector> sources_; + RTC_OBJC_TYPE(RTCDesktopMediaList)* objcMediaList_; + DesktopType type_; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_MEDIA_LIST_H_ diff --git a/sdk/objc/native/src/objc_desktop_media_list.mm b/sdk/objc/native/src/objc_desktop_media_list.mm new file mode 100644 index 0000000000..cb783737a2 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_media_list.mm @@ -0,0 +1,252 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sdk/objc/native/src/objc_desktop_media_list.h" +#include "rtc_base/checks.h" +#include "sdk/objc/native/src/objc_video_frame.h" +#include "third_party/libyuv/include/libyuv.h" + +extern "C" { +#if defined(USE_SYSTEM_LIBJPEG) +#include +#else +// Include directory supplied by gn +#include "jpeglib.h" // NOLINT +#endif +} + +#include +#include + +#import + +namespace webrtc { + +ObjCDesktopMediaList::ObjCDesktopMediaList(DesktopType type, + RTC_OBJC_TYPE(RTCDesktopMediaList) * objcMediaList) + : thread_(rtc::Thread::Create()), objcMediaList_(objcMediaList), type_(type) { + RTC_DCHECK(thread_); + thread_->Start(); + options_ = webrtc::DesktopCaptureOptions::CreateDefault(); + options_.set_detect_updated_region(true); + options_.set_allow_iosurface(true); + + callback_ = std::make_unique(); + + thread_->BlockingCall([this, type] { + if (type == kScreen) { + capturer_ = webrtc::DesktopCapturer::CreateScreenCapturer(options_); + } else { + capturer_ = webrtc::DesktopCapturer::CreateWindowCapturer(options_); + } + capturer_->Start(callback_.get()); + }); +} + +ObjCDesktopMediaList::~ObjCDesktopMediaList() { + thread_->BlockingCall([this] { + capturer_.reset(); + }); +} + +int32_t ObjCDesktopMediaList::UpdateSourceList(bool force_reload, bool get_thumbnail) { + if (force_reload) { + for (auto source : sources_) { + [objcMediaList_ mediaSourceRemoved:source.get()]; + } + sources_.clear(); + } + + webrtc::DesktopCapturer::SourceList new_sources; + + thread_->BlockingCall([this, &new_sources] { + capturer_->GetSourceList(&new_sources); + }); + + typedef std::set SourceSet; + SourceSet new_source_set; + for (size_t i = 0; i < new_sources.size(); ++i) { + if (type_ == kScreen && new_sources[i].title.length() == 0) { + new_sources[i].title = std::string("Screen " + std::to_string(i + 1)); + } + new_source_set.insert(new_sources[i].id); + } + // Iterate through the old sources to find the removed sources. + for (size_t i = 0; i < sources_.size(); ++i) { + if (new_source_set.find(sources_[i]->id()) == new_source_set.end()) { + [objcMediaList_ mediaSourceRemoved:(*(sources_.begin() + i)).get()]; + sources_.erase(sources_.begin() + i); + --i; + } + } + // Iterate through the new sources to find the added sources. + if (new_sources.size() > sources_.size()) { + SourceSet old_source_set; + for (size_t i = 0; i < sources_.size(); ++i) { + old_source_set.insert(sources_[i]->id()); + } + for (size_t i = 0; i < new_sources.size(); ++i) { + if (old_source_set.find(new_sources[i].id) == old_source_set.end()) { + MediaSource *source = new MediaSource(this, new_sources[i], type_); + sources_.insert(sources_.begin() + i, std::shared_ptr(source)); + [objcMediaList_ mediaSourceAdded:source]; + GetThumbnail(source, true); + } + } + } + + RTC_DCHECK_EQ(new_sources.size(), sources_.size()); + + // Find the moved/changed sources. + size_t pos = 0; + while (pos < sources_.size()) { + if (!(sources_[pos]->id() == new_sources[pos].id)) { + // Find the source that should be moved to |pos|, starting from |pos + 1| + // of |sources_|, because entries before |pos| should have been sorted. + size_t old_pos = pos + 1; + for (; old_pos < sources_.size(); ++old_pos) { + if (sources_[old_pos]->id() == new_sources[pos].id) break; + } + RTC_DCHECK(sources_[old_pos]->id() == new_sources[pos].id); + + // Move the source from |old_pos| to |pos|. + auto temp = sources_[old_pos]; + sources_.erase(sources_.begin() + old_pos); + sources_.insert(sources_.begin() + pos, temp); + //[objcMediaList_ mediaSourceMoved:old_pos newIndex:pos]; + } + + if (sources_[pos]->source.title != new_sources[pos].title) { + sources_[pos]->source.title = new_sources[pos].title; + [objcMediaList_ mediaSourceNameChanged:sources_[pos].get()]; + } + ++pos; + } + + if (get_thumbnail) { + for (auto source : sources_) { + GetThumbnail(source.get(), true); + } + } + return sources_.size(); +} + +bool ObjCDesktopMediaList::GetThumbnail(MediaSource *source, bool notify) { + thread_->PostTask([this, source, notify] { + if(capturer_->SelectSource(source->id())){ + callback_->SetCallback([&](webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) { + auto old_thumbnail = source->thumbnail(); + source->SaveCaptureResult(result, std::move(frame)); + if(old_thumbnail.size() != source->thumbnail().size() && notify) { + [objcMediaList_ mediaSourceThumbnailChanged:source]; + } + }); + capturer_->CaptureFrame(); + } + }); + + return true; +} + +int ObjCDesktopMediaList::GetSourceCount() const { + return sources_.size(); +} + +MediaSource *ObjCDesktopMediaList::GetSource(int index) { + return sources_[index].get(); +} + +bool MediaSource::UpdateThumbnail() { + return mediaList_->GetThumbnail(this, true); +} + +void MediaSource::SaveCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) { + if (result != webrtc::DesktopCapturer::Result::SUCCESS) { + return; + } + int width = frame->size().width(); + int height = frame->size().height(); + int real_width = width; + + if (type_ == kWindow) { + int multiple = 0; +#if defined(WEBRTC_ARCH_X86_FAMILY) + multiple = 16; +#elif defined(WEBRTC_ARCH_ARM64) + multiple = 32; +#endif + // A multiple of $multiple must be used as the width of the src frame, + // and the right black border needs to be cropped during conversion. + if (multiple != 0 && (width % multiple) != 0) { + width = (width / multiple + 1) * multiple; + } + } + + CVPixelBufferRef pixelBuffer = NULL; + + NSDictionary *pixelAttributes = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVReturn res = CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)(pixelAttributes), + &pixelBuffer); + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *pxdata = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer); + libyuv::ConvertToARGB(reinterpret_cast(frame->data()), + real_width * height * 4, + reinterpret_cast(pxdata), + width * 4, + 0, + 0, + width, + height, + real_width, + height, + libyuv::kRotate0, + libyuv::FOURCC_ARGB); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + if (res != kCVReturnSuccess) { + NSLog(@"Unable to create cvpixelbuffer %d", res); + return; + } + + CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + CGRect outputSize = CGRectMake(0, 0, width, height); + + CIContext *tempContext = [CIContext contextWithOptions:nil]; + CGImageRef cgImage = [tempContext createCGImage:ciImage fromRect:outputSize]; + NSData *imageData; + NSBitmapImageRep *newRep = [[NSBitmapImageRep alloc] initWithCGImage:cgImage]; + [newRep setSize:NSSizeToCGSize(outputSize.size)]; + imageData = [newRep representationUsingType:NSBitmapImageFileTypeJPEG + properties:@{ + NSImageCompressionFactor : @1.0f + }]; + + thumbnail_.resize(imageData.length); + const void *_Nullable rawData = [imageData bytes]; + char *src = (char *)rawData; + std::copy(src, src + imageData.length, thumbnail_.begin()); + + CGImageRelease(cgImage); + CVPixelBufferRelease(pixelBuffer); +} + +} // namespace webrtc diff --git a/sdk/objc/native/src/objc_network_monitor.h b/sdk/objc/native/src/objc_network_monitor.h index 709e9dfbe5..c5440d587b 100644 --- a/sdk/objc/native/src/objc_network_monitor.h +++ b/sdk/objc/native/src/objc_network_monitor.h @@ -59,7 +59,7 @@ class ObjCNetworkMonitor : public rtc::NetworkMonitorInterface, std::map adapter_type_by_name_ RTC_GUARDED_BY(thread_); rtc::scoped_refptr safety_flag_; - RTCNetworkMonitor* network_monitor_ = nil; + RTC_OBJC_TYPE(RTCNetworkMonitor) * network_monitor_ = nil; }; } // namespace webrtc diff --git a/sdk/objc/native/src/objc_network_monitor.mm b/sdk/objc/native/src/objc_network_monitor.mm index 535548c64c..e0785e6d0b 100644 --- a/sdk/objc/native/src/objc_network_monitor.mm +++ b/sdk/objc/native/src/objc_network_monitor.mm @@ -39,7 +39,7 @@ thread_ = rtc::Thread::Current(); RTC_DCHECK_RUN_ON(thread_); safety_flag_->SetAlive(); - network_monitor_ = [[RTCNetworkMonitor alloc] initWithObserver:this]; + network_monitor_ = [[RTC_OBJC_TYPE(RTCNetworkMonitor) alloc] initWithObserver:this]; if (network_monitor_ == nil) { RTC_LOG(LS_WARNING) << "Failed to create RTCNetworkMonitor; not available on this OS?"; } diff --git a/sdk/objc/native/src/objc_video_track_source.h b/sdk/objc/native/src/objc_video_track_source.h index 19a3d6db43..5fe39baade 100644 --- a/sdk/objc/native/src/objc_video_track_source.h +++ b/sdk/objc/native/src/objc_video_track_source.h @@ -19,7 +19,7 @@ RTC_FWD_DECL_OBJC_CLASS(RTC_OBJC_TYPE(RTCVideoFrame)); -@interface RTCObjCVideoSourceAdapter : NSObject +@interface RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter) : NSObject @end namespace webrtc { @@ -28,7 +28,7 @@ class ObjCVideoTrackSource : public rtc::AdaptedVideoTrackSource { public: ObjCVideoTrackSource(); explicit ObjCVideoTrackSource(bool is_screencast); - explicit ObjCVideoTrackSource(RTCObjCVideoSourceAdapter* adapter); + explicit ObjCVideoTrackSource(RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter)* adapter); bool is_screencast() const override; @@ -50,7 +50,7 @@ class ObjCVideoTrackSource : public rtc::AdaptedVideoTrackSource { rtc::VideoBroadcaster broadcaster_; rtc::TimestampAligner timestamp_aligner_; - RTCObjCVideoSourceAdapter* adapter_; + RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter)* adapter_; bool is_screencast_; }; diff --git a/sdk/objc/native/src/objc_video_track_source.mm b/sdk/objc/native/src/objc_video_track_source.mm index 7937e90505..401db1d111 100644 --- a/sdk/objc/native/src/objc_video_track_source.mm +++ b/sdk/objc/native/src/objc_video_track_source.mm @@ -17,11 +17,11 @@ #include "api/video/i420_buffer.h" #include "sdk/objc/native/src/objc_frame_buffer.h" -@interface RTCObjCVideoSourceAdapter () +@interface RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter) () @property(nonatomic) webrtc::ObjCVideoTrackSource *objCVideoTrackSource; @end -@implementation RTCObjCVideoSourceAdapter +@implementation RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter) @synthesize objCVideoTrackSource = _objCVideoTrackSource; @@ -40,7 +40,7 @@ - (void)capturer:(RTC_OBJC_TYPE(RTCVideoCapturer) *)capturer : AdaptedVideoTrackSource(/* required resolution alignment */ 2), is_screencast_(is_screencast) {} -ObjCVideoTrackSource::ObjCVideoTrackSource(RTCObjCVideoSourceAdapter *adapter) : adapter_(adapter) { +ObjCVideoTrackSource::ObjCVideoTrackSource(RTC_OBJC_TYPE(RTCObjCVideoSourceAdapter) *adapter) : adapter_(adapter) { adapter_.objCVideoTrackSource = this; } diff --git a/sdk/objc/unittests/RTCMTLVideoView_xctest.m b/sdk/objc/unittests/RTCMTLVideoView_xctest.m index 587a6b588f..84d73586e2 100644 --- a/sdk/objc/unittests/RTCMTLVideoView_xctest.m +++ b/sdk/objc/unittests/RTCMTLVideoView_xctest.m @@ -32,8 +32,8 @@ @interface RTC_OBJC_TYPE (RTCMTLVideoView) + (BOOL)isMetalAvailable; + (UIView *)createMetalView:(CGRect)frame; -+ (id)createNV12Renderer; -+ (id)createI420Renderer; ++ (id)createNV12Renderer; ++ (id)createI420Renderer; - (void)drawInMTKView:(id)view; @end @@ -91,7 +91,7 @@ - (id)frameMockWithCVPixelBuffer:(BOOL)hasCVPixelBuffer { } - (id)rendererMockWithSuccessfulSetup:(BOOL)success { - id rendererMock = OCMClassMock([RTCMTLRenderer class]); + id rendererMock = OCMClassMock([RTC_OBJC_TYPE(RTCMTLRenderer) class]); OCMStub([rendererMock addRenderingDestination:[OCMArg any]]).andReturn(success); return rendererMock; } diff --git a/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm b/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm index 5ba5a52a53..c4dda5aef1 100644 --- a/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm +++ b/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm @@ -46,7 +46,7 @@ - (void)testBuilder { nativeVideoDecoderFactory:nullptr audioDeviceModule:nullptr audioProcessingModule:nullptr]); - RTCPeerConnectionFactoryBuilder* builder = [[RTCPeerConnectionFactoryBuilder alloc] init]; + RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder)* builder = [[RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) alloc] init]; RTC_OBJC_TYPE(RTCPeerConnectionFactory)* peerConnectionFactory = [builder createPeerConnectionFactory]; EXPECT_TRUE(peerConnectionFactory != nil); @@ -63,7 +63,7 @@ - (void)testDefaultComponentsBuilder { nativeVideoDecoderFactory:nullptr audioDeviceModule:nullptr audioProcessingModule:nullptr]); - RTCPeerConnectionFactoryBuilder* builder = [RTCPeerConnectionFactoryBuilder defaultBuilder]; + RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder)* builder = [RTC_OBJC_TYPE(RTCPeerConnectionFactoryBuilder) defaultBuilder]; RTC_OBJC_TYPE(RTCPeerConnectionFactory)* peerConnectionFactory = [builder createPeerConnectionFactory]; EXPECT_TRUE(peerConnectionFactory != nil); diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 5ed5110c96..dc0d34e720 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -2030,9 +2030,15 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, was_encode_called_since_last_initialization_ = true; if (encode_status < 0) { - RTC_LOG(LS_ERROR) << "Encoder failed, failing encoder format: " - << encoder_config_.video_format.ToString(); - RequestEncoderSwitch(); + if (encode_status == WEBRTC_VIDEO_CODEC_ENCODER_FAILURE) { + RTC_LOG(LS_ERROR) << "Encoder failed, failing encoder format: " + << encoder_config_.video_format.ToString(); + RequestEncoderSwitch(); + } else { + RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " + << encode_status; + } + return; }