diff --git a/modules/desktop_capture/mac/screen_capturer_mac.mm b/modules/desktop_capture/mac/screen_capturer_mac.mm index 60089fd0f2..aa05ceb9aa 100644 --- a/modules/desktop_capture/mac/screen_capturer_mac.mm +++ b/modules/desktop_capture/mac/screen_capturer_mac.mm @@ -216,16 +216,7 @@ DesktopRect GetExcludedWindowPixelBounds(CGWindowID window, float dip_to_pixel_s ScreenConfigurationChanged(); } - // When screen is zoomed in/out, OSX only updates the part of Rects currently - // displayed on screen, with relative location to current top-left on screen. - // This will cause problems when we copy the dirty regions to the captured - // image. So we invalidate the whole screen to copy all the screen contents. - // With CGI method, the zooming will be ignored and the whole screen contents - // will be captured as before. - // With IOSurface method, the zoomed screen contents will be captured. - if (UAZoomEnabled()) { - helper_.InvalidateScreen(screen_pixel_bounds_.size()); - } + helper_.InvalidateScreen(screen_pixel_bounds_.size()); DesktopRegion region; helper_.TakeInvalidRegion(®ion); diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 04063216cc..691d6011e8 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -682,6 +682,43 @@ if (is_ios || is_mac) { "../rtc_base/system:gcd_helpers", ] } + + rtc_library("desktopcapture_objc") { + visibility = [ "*" ] + sources = [ + "objc/components/capturer/RTCDesktopCapturer+Private.h", + "objc/components/capturer/RTCDesktopCapturer.h", + "objc/components/capturer/RTCDesktopCapturer.mm", + "objc/components/capturer/RTCDesktopSource+Private.h", + "objc/components/capturer/RTCDesktopSource.h", + "objc/components/capturer/RTCDesktopSource.mm", + "objc/components/capturer/RTCDesktopMediaList+Private.h", + "objc/components/capturer/RTCDesktopMediaList.h", + "objc/components/capturer/RTCDesktopMediaList.mm", + "objc/native/src/objc_desktop_capture.h", + "objc/native/src/objc_desktop_capture.mm", + "objc/native/src/objc_desktop_media_list.h", + "objc/native/src/objc_desktop_media_list.mm", + ] + frameworks = [ + "AppKit.framework", + ] + + configs += [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":helpers_objc", + ":videoframebuffer_objc", + "../rtc_base/system:gcd_helpers", + "../modules/desktop_capture", + ] + if(is_mac) { + deps += [ "//third_party:jpeg", ] + } + } rtc_library("videocodec_objc") { visibility = [ "*" ] @@ -1534,6 +1571,9 @@ if (is_ios || is_mac) { "objc/base/RTCYUVPlanarBuffer.h", "objc/components/capturer/RTCCameraVideoCapturer.h", "objc/components/capturer/RTCFileVideoCapturer.h", + "objc/components/capturer/RTCDesktopCapturer.h", + "objc/components/capturer/RTCDesktopSource.h", + "objc/components/capturer/RTCDesktopMediaList.h", "objc/components/renderer/metal/RTCMTLVideoView.h", "objc/components/renderer/metal/RTCMTLNSVideoView.h", "objc/components/renderer/opengl/RTCVideoViewShading.h", @@ -1566,6 +1606,7 @@ if (is_ios || is_mac) { ":native_video", ":peerconnectionfactory_base_objc", ":videocapture_objc", + ":desktopcapture_objc", ":videocodec_objc", ":videotoolbox_objc", ":darwin_privacy_info", diff --git a/sdk/objc/components/capturer/RTCCameraVideoCapturer.m b/sdk/objc/components/capturer/RTCCameraVideoCapturer.m index e7c47b4e99..d7c698434e 100644 --- a/sdk/objc/components/capturer/RTCCameraVideoCapturer.m +++ b/sdk/objc/components/capturer/RTCCameraVideoCapturer.m @@ -490,7 +490,9 @@ - (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger @"updateDeviceCaptureFormat must be called on the capture queue."); @try { _currentDevice.activeFormat = format; - _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); + if(![NSStringFromClass([_currentDevice class]) isEqualToString:@"AVCaptureDALDevice"]) { + _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); + } } @catch (NSException *exception) { RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo); return; diff --git a/sdk/objc/components/capturer/RTCDesktopCapturer+Private.h b/sdk/objc/components/capturer/RTCDesktopCapturer+Private.h new file mode 100644 index 0000000000..7e293a1a56 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopCapturer+Private.h @@ -0,0 +1,49 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDesktopCapturer.h" + +#include "sdk/objc/native/src/objc_desktop_capture.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(DesktopCapturerDelegate) +-(void)didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *) frame; +-(void)didSourceCaptureStart; +-(void)didSourceCapturePaused; +-(void)didSourceCaptureStop; +-(void)didSourceCaptureError; +@end + +@interface RTCDesktopCapturer () + +@property(nonatomic, readonly)std::shared_ptr nativeCapturer; + +- (void)didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +-(void)didSourceCaptureStart; + +-(void)didSourceCapturePaused; + +-(void)didSourceCaptureStop; + +-(void)didSourceCaptureError; + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopCapturer.h b/sdk/objc/components/capturer/RTCDesktopCapturer.h new file mode 100644 index 0000000000..160c00d208 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopCapturer.h @@ -0,0 +1,61 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import + +#import "RTCMacros.h" +#import "RTCVideoCapturer.h" +#import "RTCDesktopSource.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTCDesktopCapturer; + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCDesktopCapturerDelegate) +-(void)didSourceCaptureStart:(RTCDesktopCapturer *) capturer; + +-(void)didSourceCapturePaused:(RTCDesktopCapturer *) capturer; + +-(void)didSourceCaptureStop:(RTCDesktopCapturer *) capturer; + +-(void)didSourceCaptureError:(RTCDesktopCapturer *) capturer; +@end + +RTC_OBJC_EXPORT +// Screen capture that implements RTCVideoCapturer. Delivers frames to a +// RTCVideoCapturerDelegate (usually RTCVideoSource). +@interface RTC_OBJC_TYPE (RTCDesktopCapturer) : RTC_OBJC_TYPE(RTCVideoCapturer) + +@property(nonatomic, readonly) RTCDesktopSource *source; + +- (instancetype)initWithSource:(RTCDesktopSource*)source delegate:(__weak id)delegate captureDelegate:(__weak id)captureDelegate; + +- (instancetype)initWithDefaultScreen:(__weak id)delegate captureDelegate:(__weak id)captureDelegate; + +- (void)startCapture; + +- (void)startCaptureWithFPS:(NSInteger)fps; + +- (void)stopCapture; + +- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/capturer/RTCDesktopCapturer.mm b/sdk/objc/components/capturer/RTCDesktopCapturer.mm new file mode 100644 index 0000000000..a7d5c60eb2 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopCapturer.mm @@ -0,0 +1,104 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "base/RTCLogging.h" +#import "base/RTCVideoFrameBuffer.h" + +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#import "RTCDesktopCapturer.h" +#import "RTCDesktopCapturer+Private.h" +#import "RTCDesktopSource+Private.h" + +@implementation RTC_OBJC_TYPE (RTCDesktopCapturer) { + __weak id _delegate; +} + +@synthesize nativeCapturer = _nativeCapturer; +@synthesize source = _source; + +- (instancetype)initWithSource:(RTCDesktopSource*)source delegate:(__weak id)delegate captureDelegate:(__weak id)captureDelegate { + if (self = [super initWithDelegate:captureDelegate]) { + webrtc::DesktopType captureType = webrtc::kScreen; + if(source.sourceType == RTCDesktopSourceTypeWindow) { + captureType = webrtc::kWindow; + } + _nativeCapturer = std::make_shared(captureType, source.nativeMediaSource->id(), self); + _source = source; + _delegate = delegate; + } + return self; +} + +- (instancetype)initWithDefaultScreen:(__weak id)delegate captureDelegate:(__weak id)captureDelegate { + if (self = [super initWithDelegate:captureDelegate]) { + _nativeCapturer = std::make_unique(webrtc::kScreen, -1, self); + _source = nil; + _delegate = delegate; + } + return self; +} + + +-(void)dealloc { + _nativeCapturer->Stop(); + _nativeCapturer = nullptr; +} + +- (void)startCapture { + [self didSourceCaptureStart]; + _nativeCapturer->Start(30); +} + +- (void)startCaptureWithFPS:(NSInteger)fps { + _nativeCapturer->Start(fps); +} + +- (void)didCaptureVideoFrame + : (RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + [self.delegate capturer:self didCaptureVideoFrame:frame]; +} + +- (void)stopCapture { + _nativeCapturer->Stop(); +} + +- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler { + [self stopCapture]; + if(completionHandler != nil) { + completionHandler(); + } +} + +-(void)didSourceCaptureStart { + [_delegate didSourceCaptureStart:self]; +} + +-(void)didSourceCapturePaused { + [_delegate didSourceCapturePaused:self]; +} + +-(void)didSourceCaptureStop { + [_delegate didSourceCaptureStop:self]; +} + +-(void)didSourceCaptureError { + [_delegate didSourceCaptureError:self]; +} + +@end diff --git a/sdk/objc/components/capturer/RTCDesktopMediaList+Private.h b/sdk/objc/components/capturer/RTCDesktopMediaList+Private.h new file mode 100644 index 0000000000..fc3b080ad2 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopMediaList+Private.h @@ -0,0 +1,40 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDesktopMediaList.h" + +namespace webrtc { + class ObjCDesktopMediaList; + class MediaSource; +} + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCDesktopMediaList () + +@property(nonatomic, readonly)std::shared_ptr nativeMediaList; + +-(void)mediaSourceAdded:(webrtc::MediaSource *) source; + +-(void)mediaSourceRemoved:(webrtc::MediaSource *) source; + +-(void)mediaSourceNameChanged:(webrtc::MediaSource *) source; + +-(void)mediaSourceThumbnailChanged:(webrtc::MediaSource *) source; + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopMediaList.h b/sdk/objc/components/capturer/RTCDesktopMediaList.h new file mode 100644 index 0000000000..fafeaf5e0d --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopMediaList.h @@ -0,0 +1,51 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import + +#import "RTCMacros.h" +#import "RTCDesktopSource.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCDesktopMediaListDelegate) + +- (void)didDesktopSourceAdded:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; + +- (void)didDesktopSourceRemoved:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; + +- (void)didDesktopSourceNameChanged:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; + +- (void)didDesktopSourceThumbnailChanged:(RTC_OBJC_TYPE(RTCDesktopSource) *) source; +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDesktopMediaList) : NSObject + +-(instancetype)initWithType:(RTCDesktopSourceType)type delegate:(__weak id)delegate; + +@property(nonatomic, readonly) RTCDesktopSourceType sourceType; + +- (int32_t)UpdateSourceList:(BOOL)forceReload updateAllThumbnails:(BOOL)updateThumbnail; + +- (NSArray*) getSources; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/capturer/RTCDesktopMediaList.mm b/sdk/objc/components/capturer/RTCDesktopMediaList.mm new file mode 100644 index 0000000000..7aa7dca1be --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopMediaList.mm @@ -0,0 +1,99 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDesktopMediaList.h" + +#import "RTCDesktopSource+Private.h" +#import "RTCDesktopMediaList+Private.h" + +@implementation RTCDesktopMediaList { + RTCDesktopSourceType _sourceType; + NSMutableArray* _sources; + __weak id _delegate; +} + +@synthesize sourceType = _sourceType; +@synthesize nativeMediaList = _nativeMediaList; + +- (instancetype)initWithType:(RTCDesktopSourceType)type delegate:(__weak id)delegate{ + if (self = [super init]) { + webrtc::DesktopType captureType = webrtc::kScreen; + if(type == RTCDesktopSourceTypeWindow) { + captureType = webrtc::kWindow; + } + _nativeMediaList = std::make_shared(captureType, self); + _sourceType = type; + _delegate = delegate; + } + return self; +} + +- (int32_t)UpdateSourceList:(BOOL)forceReload updateAllThumbnails:(BOOL)updateThumbnail { + return _nativeMediaList->UpdateSourceList(forceReload, updateThumbnail); +} + +-(NSArray*) getSources { + _sources = [NSMutableArray array]; + int sourceCount = _nativeMediaList->GetSourceCount(); + for (int i = 0; i < sourceCount; i++) { + webrtc::MediaSource *mediaSource = _nativeMediaList->GetSource(i); + [_sources addObject:[[RTCDesktopSource alloc] initWithNativeSource:mediaSource sourceType:_sourceType]]; + } + return _sources; +} + +-(void)mediaSourceAdded:(webrtc::MediaSource *) source { + RTCDesktopSource *desktopSource = [[RTCDesktopSource alloc] initWithNativeSource:source sourceType:_sourceType]; + [_sources addObject:desktopSource]; + [_delegate didDesktopSourceAdded:desktopSource]; +} + +-(void)mediaSourceRemoved:(webrtc::MediaSource *) source { + RTCDesktopSource *desktopSource = [self getSourceById:source]; + if(desktopSource != nil) { + [_sources removeObject:desktopSource]; + [_delegate didDesktopSourceRemoved:desktopSource]; + } +} + +-(void)mediaSourceNameChanged:(webrtc::MediaSource *) source { + RTCDesktopSource *desktopSource = [self getSourceById:source]; + if(desktopSource != nil) { + [desktopSource setName:source->name().c_str()]; + [_delegate didDesktopSourceNameChanged:desktopSource]; + } +} + +-(void)mediaSourceThumbnailChanged:(webrtc::MediaSource *) source { + RTCDesktopSource *desktopSource = [self getSourceById:source]; + if(desktopSource != nil) { + [desktopSource setThumbnail:source->thumbnail()]; + [_delegate didDesktopSourceThumbnailChanged:desktopSource]; + } +} + +-(RTCDesktopSource *)getSourceById:(webrtc::MediaSource *) source { + NSEnumerator *enumerator = [_sources objectEnumerator]; + RTCDesktopSource *object; + while ((object = enumerator.nextObject) != nil) { + if(object.nativeMediaSource == source) { + return object; + } + } + return nil; +} + +@end \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopSource+Private.h b/sdk/objc/components/capturer/RTCDesktopSource+Private.h new file mode 100644 index 0000000000..f5a0d14e0f --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopSource+Private.h @@ -0,0 +1,37 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#import + +#import "RTCDesktopSource.h" + +#include "sdk/objc/native/src/objc_desktop_media_list.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCDesktopSource () + +- (instancetype)initWithNativeSource:(webrtc::MediaSource*) nativeSource + sourceType:(RTCDesktopSourceType) sourceType; + +@property(nonatomic, readonly)webrtc::MediaSource* nativeMediaSource; + +-(void) setName:(const char *) name; + +-(void) setThumbnail:(std::vector) thumbnail; + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopSource.h b/sdk/objc/components/capturer/RTCDesktopSource.h new file mode 100644 index 0000000000..82da458ce6 --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopSource.h @@ -0,0 +1,40 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#import +#import +#import + +#import "RTCMacros.h" + +typedef NS_ENUM(NSInteger, RTCDesktopSourceType) { + RTCDesktopSourceTypeScreen, + RTCDesktopSourceTypeWindow, +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDesktopSource) : NSObject + +@property(nonatomic, readonly) NSString *sourceId; + +@property(nonatomic, readonly) NSString *name; + +@property(nonatomic, readonly) NSImage *thumbnail; + +@property(nonatomic, readonly) RTCDesktopSourceType sourceType; + +-( NSImage *)UpdateThumbnail; + +@end \ No newline at end of file diff --git a/sdk/objc/components/capturer/RTCDesktopSource.mm b/sdk/objc/components/capturer/RTCDesktopSource.mm new file mode 100644 index 0000000000..3f5f23894b --- /dev/null +++ b/sdk/objc/components/capturer/RTCDesktopSource.mm @@ -0,0 +1,68 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCDesktopSource.h" +#import "RTCDesktopSource+Private.h" + +@implementation RTCDesktopSource { + NSString *_sourceId; + NSString *_name; + NSImage *_thumbnail; + RTCDesktopSourceType _sourceType; +} + +@synthesize sourceId = _sourceId; +@synthesize name = _name; +@synthesize thumbnail = _thumbnail; +@synthesize sourceType = _sourceType; +@synthesize nativeMediaSource = _nativeMediaSource; + +- (instancetype)initWithNativeSource:(webrtc::MediaSource*)nativeSource + sourceType:(RTCDesktopSourceType) sourceType { + if (self = [super init]) { + _nativeMediaSource = nativeSource; + _sourceId = [NSString stringWithUTF8String:std::to_string(nativeSource->id()).c_str()]; + _name = [NSString stringWithUTF8String:nativeSource->name().c_str()]; + _thumbnail = [self createThumbnailFromNativeSource:nativeSource->thumbnail()]; + _sourceType = sourceType; + } + return self; +} + +-(NSImage*)createThumbnailFromNativeSource:(std::vector)thumbnail { + NSData* data = [[NSData alloc] initWithBytes:thumbnail.data() length:thumbnail.size()]; + NSImage *image = [[NSImage alloc] initWithData:data]; + return image; +} + +-( NSImage *)UpdateThumbnail { + if(_nativeMediaSource->UpdateThumbnail()) { + _thumbnail = [self createThumbnailFromNativeSource:_nativeMediaSource->thumbnail()]; + } + return _thumbnail; +} + +-(void)setName:(const char *) name { + _name = [NSString stringWithUTF8String:name]; +} + +-(void)setThumbnail:(std::vector) thumbnail { + _thumbnail = [self createThumbnailFromNativeSource:thumbnail]; +} + +@end diff --git a/sdk/objc/native/src/objc_desktop_capture.h b/sdk/objc/native/src/objc_desktop_capture.h new file mode 100644 index 0000000000..a781457220 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_capture.h @@ -0,0 +1,70 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_CAPTURE_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_CAPTURE_H_ + +#import "base/RTCMacros.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_and_cursor_composer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "rtc_base/thread.h" + +@protocol RTC_OBJC_TYPE +(DesktopCapturerDelegate); + +namespace webrtc { + +enum DesktopType { kScreen, kWindow }; + +class ObjCDesktopCapturer : public DesktopCapturer::Callback { + public: + enum CaptureState { CS_RUNNING, CS_STOPPED, CS_FAILED}; + + public: + ObjCDesktopCapturer(DesktopType type, + webrtc::DesktopCapturer::SourceId source_id, + id delegate); + virtual ~ObjCDesktopCapturer(); + + virtual CaptureState Start(uint32_t fps); + + virtual void Stop(); + + virtual bool IsRunning(); + + protected: + virtual void OnCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) override; + private: + void CaptureFrame(); + webrtc::DesktopCaptureOptions options_; + std::unique_ptr capturer_; + std::unique_ptr thread_; + CaptureState capture_state_ = CS_STOPPED; + DesktopType type_; + webrtc::DesktopCapturer::SourceId source_id_; + id delegate_; + uint32_t capture_delay_ = 1000; // 1s + webrtc::DesktopCapturer::Result result_ = webrtc::DesktopCapturer::Result::SUCCESS; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_CAPTURE_H_ diff --git a/sdk/objc/native/src/objc_desktop_capture.mm b/sdk/objc/native/src/objc_desktop_capture.mm new file mode 100644 index 0000000000..7aba3e5612 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_capture.mm @@ -0,0 +1,205 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/sequence_checker.h" +#include "rtc_base/checks.h" +#include "sdk/objc/native/src/objc_desktop_capture.h" +#include "sdk/objc/native/src/objc_video_frame.h" +#include "third_party/libyuv/include/libyuv.h" + +#import "components/capturer/RTCDesktopCapturer+Private.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +namespace webrtc { + +enum { kCaptureDelay = 33, kCaptureMessageId = 1000 }; + +ObjCDesktopCapturer::ObjCDesktopCapturer(DesktopType type, + webrtc::DesktopCapturer::SourceId source_id, + id delegate) + : thread_(rtc::Thread::Create()), source_id_(source_id), delegate_(delegate) { + RTC_DCHECK(thread_); + type_ = type; + thread_->Start(); + options_ = webrtc::DesktopCaptureOptions::CreateDefault(); + options_.set_detect_updated_region(true); + options_.set_allow_iosurface(true); + thread_->BlockingCall([this, type] { + if (type == kScreen) { + capturer_ = std::make_unique( + webrtc::DesktopCapturer::CreateScreenCapturer(options_), options_); + } else { + capturer_ = std::make_unique( + webrtc::DesktopCapturer::CreateWindowCapturer(options_), options_); + } + }); +} + +ObjCDesktopCapturer::~ObjCDesktopCapturer() { + thread_->BlockingCall([this] { + capturer_.reset(); + }); +} + +ObjCDesktopCapturer::CaptureState ObjCDesktopCapturer::Start(uint32_t fps) { + if(capture_state_ == CS_RUNNING) { + return capture_state_; + } + + if(fps == 0) { + capture_state_ = CS_FAILED; + return capture_state_; + } + + if (fps >= 60) { + capture_delay_ = uint32_t(1000.0 / 60.0); + } else { + capture_delay_ = uint32_t(1000.0 / fps); + } + + if (source_id_ != -1) { + if (!capturer_->SelectSource(source_id_)) { + capture_state_ = CS_FAILED; + return capture_state_; + } + if (type_ == kWindow) { + if (!capturer_->FocusOnSelectedSource()) { + capture_state_ = CS_FAILED; + return capture_state_; + } + } + } + + thread_->BlockingCall([this] { + capturer_->Start(this); + }); + capture_state_ = CS_RUNNING; + + thread_->PostTask([this] { + CaptureFrame(); + }); + + [delegate_ didSourceCaptureStart]; + return capture_state_; +} + +void ObjCDesktopCapturer::Stop() { + [delegate_ didSourceCaptureStop]; + capture_state_ = CS_STOPPED; +} + +bool ObjCDesktopCapturer::IsRunning() { + return capture_state_ == CS_RUNNING; +} + +void ObjCDesktopCapturer::OnCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) { + if (result != result_) { + if (result == webrtc::DesktopCapturer::Result::ERROR_PERMANENT) { + [delegate_ didSourceCaptureError]; + capture_state_ = CS_FAILED; + return; + } + + if (result == webrtc::DesktopCapturer::Result::ERROR_TEMPORARY) { + result_ = result; + [delegate_ didSourceCapturePaused]; + return; + } + + if (result == webrtc::DesktopCapturer::Result::SUCCESS) { + result_ = result; + [delegate_ didSourceCaptureStart]; + } + } + + if (result == webrtc::DesktopCapturer::Result::ERROR_TEMPORARY) { + return; + } + + int width = frame->size().width(); + int height = frame->size().height(); + int real_width = width; + + if (type_ == kWindow) { + int multiple = 0; +#if defined(WEBRTC_ARCH_X86_FAMILY) + multiple = 16; +#elif defined(WEBRTC_ARCH_ARM64) + multiple = 32; +#endif + // A multiple of $multiple must be used as the width of the src frame, + // and the right black border needs to be cropped during conversion. + if (multiple != 0 && (width % multiple) != 0) { + width = (width / multiple + 1) * multiple; + } + } + + CVPixelBufferRef pixelBuffer = NULL; + + NSDictionary *pixelAttributes = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVReturn res = CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)(pixelAttributes), + &pixelBuffer); + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *pxdata = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer); + libyuv::ConvertToARGB(reinterpret_cast(frame->data()), + real_width * height * 4, + reinterpret_cast(pxdata), + width * 4, + 0, + 0, + width, + height, + real_width, + height, + libyuv::kRotate0, + libyuv::FOURCC_ARGB); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + if (res != kCVReturnSuccess) { + NSLog(@"Unable to create cvpixelbuffer %d", res); + return; + } + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer]; + NSTimeInterval timeStampSeconds = CACurrentMediaTime(); + int64_t timeStampNs = lroundf(timeStampSeconds * NSEC_PER_SEC); + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer + rotation:RTCVideoRotation_0 + timeStampNs:timeStampNs]; + CVPixelBufferRelease(pixelBuffer); + [delegate_ didCaptureVideoFrame:videoFrame]; +} + +void ObjCDesktopCapturer::CaptureFrame() { + RTC_DCHECK_RUN_ON(thread_.get()); + if (capture_state_ == CS_RUNNING) { + capturer_->CaptureFrame(); + thread_->PostDelayedHighPrecisionTask( + [this]() { + CaptureFrame(); + }, + TimeDelta::Millis(capture_delay_)); + } +} + +} // namespace webrtc diff --git a/sdk/objc/native/src/objc_desktop_media_list.h b/sdk/objc/native/src/objc_desktop_media_list.h new file mode 100644 index 0000000000..ecb2d27221 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_media_list.h @@ -0,0 +1,111 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_MEDIA_LIST_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_MEDIA_LIST_H_ + +#import "base/RTCMacros.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "rtc_base/thread.h" + +#include "objc_desktop_capture.h" + +#import "components/capturer/RTCDesktopMediaList+Private.h" + +namespace webrtc { + +class MediaSource { + public: + MediaSource( ObjCDesktopMediaList *mediaList, DesktopCapturer::Source src, DesktopType type) + : source(src), mediaList_(mediaList), type_(type) {} + virtual ~MediaSource() {} + + DesktopCapturer::Source source; + + // source id + DesktopCapturer::SourceId id() const { return source.id; } + + // source name + std::string name() const { return source.title; } + + // Returns the thumbnail of the source, jpeg format. + std::vector thumbnail() const { return thumbnail_; } + + + + DesktopType type() const { return type_; } + + bool UpdateThumbnail(); + + void SaveCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame); + + private: + std::vector thumbnail_; + ObjCDesktopMediaList *mediaList_; + DesktopType type_; +}; + +class ObjCDesktopMediaList { + public: + enum CaptureState { CS_RUNNING, CS_STOPPED, CS_FAILED}; + public: + ObjCDesktopMediaList(DesktopType type, RTC_OBJC_TYPE(RTCDesktopMediaList)* objcMediaList); + + virtual ~ObjCDesktopMediaList(); + + virtual int32_t UpdateSourceList(bool force_reload = false, bool get_thumbnail = true); + + virtual int GetSourceCount() const; + + virtual MediaSource* GetSource(int index); + + virtual bool GetThumbnail(MediaSource *source, bool notify); + + private: + class CallbackProxy : public DesktopCapturer::Callback { + public: + CallbackProxy(){} + void SetCallback(std::function frame)> on_capture_result) { + on_capture_result_ = on_capture_result; + } + private: + void OnCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) override { + if(on_capture_result_) on_capture_result_(result, std::move(frame)); + } + std::function frame)> on_capture_result_ = nullptr; + }; + private: + std::unique_ptr callback_; + webrtc::DesktopCaptureOptions options_; + std::unique_ptr capturer_; + std::unique_ptr thread_; + std::vector> sources_; + RTC_OBJC_TYPE(RTCDesktopMediaList)* objcMediaList_; + DesktopType type_; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_DESKTOP_MEDIA_LIST_H_ diff --git a/sdk/objc/native/src/objc_desktop_media_list.mm b/sdk/objc/native/src/objc_desktop_media_list.mm new file mode 100644 index 0000000000..cb783737a2 --- /dev/null +++ b/sdk/objc/native/src/objc_desktop_media_list.mm @@ -0,0 +1,252 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sdk/objc/native/src/objc_desktop_media_list.h" +#include "rtc_base/checks.h" +#include "sdk/objc/native/src/objc_video_frame.h" +#include "third_party/libyuv/include/libyuv.h" + +extern "C" { +#if defined(USE_SYSTEM_LIBJPEG) +#include +#else +// Include directory supplied by gn +#include "jpeglib.h" // NOLINT +#endif +} + +#include +#include + +#import + +namespace webrtc { + +ObjCDesktopMediaList::ObjCDesktopMediaList(DesktopType type, + RTC_OBJC_TYPE(RTCDesktopMediaList) * objcMediaList) + : thread_(rtc::Thread::Create()), objcMediaList_(objcMediaList), type_(type) { + RTC_DCHECK(thread_); + thread_->Start(); + options_ = webrtc::DesktopCaptureOptions::CreateDefault(); + options_.set_detect_updated_region(true); + options_.set_allow_iosurface(true); + + callback_ = std::make_unique(); + + thread_->BlockingCall([this, type] { + if (type == kScreen) { + capturer_ = webrtc::DesktopCapturer::CreateScreenCapturer(options_); + } else { + capturer_ = webrtc::DesktopCapturer::CreateWindowCapturer(options_); + } + capturer_->Start(callback_.get()); + }); +} + +ObjCDesktopMediaList::~ObjCDesktopMediaList() { + thread_->BlockingCall([this] { + capturer_.reset(); + }); +} + +int32_t ObjCDesktopMediaList::UpdateSourceList(bool force_reload, bool get_thumbnail) { + if (force_reload) { + for (auto source : sources_) { + [objcMediaList_ mediaSourceRemoved:source.get()]; + } + sources_.clear(); + } + + webrtc::DesktopCapturer::SourceList new_sources; + + thread_->BlockingCall([this, &new_sources] { + capturer_->GetSourceList(&new_sources); + }); + + typedef std::set SourceSet; + SourceSet new_source_set; + for (size_t i = 0; i < new_sources.size(); ++i) { + if (type_ == kScreen && new_sources[i].title.length() == 0) { + new_sources[i].title = std::string("Screen " + std::to_string(i + 1)); + } + new_source_set.insert(new_sources[i].id); + } + // Iterate through the old sources to find the removed sources. + for (size_t i = 0; i < sources_.size(); ++i) { + if (new_source_set.find(sources_[i]->id()) == new_source_set.end()) { + [objcMediaList_ mediaSourceRemoved:(*(sources_.begin() + i)).get()]; + sources_.erase(sources_.begin() + i); + --i; + } + } + // Iterate through the new sources to find the added sources. + if (new_sources.size() > sources_.size()) { + SourceSet old_source_set; + for (size_t i = 0; i < sources_.size(); ++i) { + old_source_set.insert(sources_[i]->id()); + } + for (size_t i = 0; i < new_sources.size(); ++i) { + if (old_source_set.find(new_sources[i].id) == old_source_set.end()) { + MediaSource *source = new MediaSource(this, new_sources[i], type_); + sources_.insert(sources_.begin() + i, std::shared_ptr(source)); + [objcMediaList_ mediaSourceAdded:source]; + GetThumbnail(source, true); + } + } + } + + RTC_DCHECK_EQ(new_sources.size(), sources_.size()); + + // Find the moved/changed sources. + size_t pos = 0; + while (pos < sources_.size()) { + if (!(sources_[pos]->id() == new_sources[pos].id)) { + // Find the source that should be moved to |pos|, starting from |pos + 1| + // of |sources_|, because entries before |pos| should have been sorted. + size_t old_pos = pos + 1; + for (; old_pos < sources_.size(); ++old_pos) { + if (sources_[old_pos]->id() == new_sources[pos].id) break; + } + RTC_DCHECK(sources_[old_pos]->id() == new_sources[pos].id); + + // Move the source from |old_pos| to |pos|. + auto temp = sources_[old_pos]; + sources_.erase(sources_.begin() + old_pos); + sources_.insert(sources_.begin() + pos, temp); + //[objcMediaList_ mediaSourceMoved:old_pos newIndex:pos]; + } + + if (sources_[pos]->source.title != new_sources[pos].title) { + sources_[pos]->source.title = new_sources[pos].title; + [objcMediaList_ mediaSourceNameChanged:sources_[pos].get()]; + } + ++pos; + } + + if (get_thumbnail) { + for (auto source : sources_) { + GetThumbnail(source.get(), true); + } + } + return sources_.size(); +} + +bool ObjCDesktopMediaList::GetThumbnail(MediaSource *source, bool notify) { + thread_->PostTask([this, source, notify] { + if(capturer_->SelectSource(source->id())){ + callback_->SetCallback([&](webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) { + auto old_thumbnail = source->thumbnail(); + source->SaveCaptureResult(result, std::move(frame)); + if(old_thumbnail.size() != source->thumbnail().size() && notify) { + [objcMediaList_ mediaSourceThumbnailChanged:source]; + } + }); + capturer_->CaptureFrame(); + } + }); + + return true; +} + +int ObjCDesktopMediaList::GetSourceCount() const { + return sources_.size(); +} + +MediaSource *ObjCDesktopMediaList::GetSource(int index) { + return sources_[index].get(); +} + +bool MediaSource::UpdateThumbnail() { + return mediaList_->GetThumbnail(this, true); +} + +void MediaSource::SaveCaptureResult(webrtc::DesktopCapturer::Result result, + std::unique_ptr frame) { + if (result != webrtc::DesktopCapturer::Result::SUCCESS) { + return; + } + int width = frame->size().width(); + int height = frame->size().height(); + int real_width = width; + + if (type_ == kWindow) { + int multiple = 0; +#if defined(WEBRTC_ARCH_X86_FAMILY) + multiple = 16; +#elif defined(WEBRTC_ARCH_ARM64) + multiple = 32; +#endif + // A multiple of $multiple must be used as the width of the src frame, + // and the right black border needs to be cropped during conversion. + if (multiple != 0 && (width % multiple) != 0) { + width = (width / multiple + 1) * multiple; + } + } + + CVPixelBufferRef pixelBuffer = NULL; + + NSDictionary *pixelAttributes = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVReturn res = CVPixelBufferCreate(kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)(pixelAttributes), + &pixelBuffer); + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + uint8_t *pxdata = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer); + libyuv::ConvertToARGB(reinterpret_cast(frame->data()), + real_width * height * 4, + reinterpret_cast(pxdata), + width * 4, + 0, + 0, + width, + height, + real_width, + height, + libyuv::kRotate0, + libyuv::FOURCC_ARGB); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + if (res != kCVReturnSuccess) { + NSLog(@"Unable to create cvpixelbuffer %d", res); + return; + } + + CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + CGRect outputSize = CGRectMake(0, 0, width, height); + + CIContext *tempContext = [CIContext contextWithOptions:nil]; + CGImageRef cgImage = [tempContext createCGImage:ciImage fromRect:outputSize]; + NSData *imageData; + NSBitmapImageRep *newRep = [[NSBitmapImageRep alloc] initWithCGImage:cgImage]; + [newRep setSize:NSSizeToCGSize(outputSize.size)]; + imageData = [newRep representationUsingType:NSBitmapImageFileTypeJPEG + properties:@{ + NSImageCompressionFactor : @1.0f + }]; + + thumbnail_.resize(imageData.length); + const void *_Nullable rawData = [imageData bytes]; + char *src = (char *)rawData; + std::copy(src, src + imageData.length, thumbnail_.begin()); + + CGImageRelease(cgImage); + CVPixelBufferRelease(pixelBuffer); +} + +} // namespace webrtc diff --git a/sdk/objc/native/src/objc_video_track_source.mm b/sdk/objc/native/src/objc_video_track_source.mm index 7937e90505..51b4d4e6b4 100644 --- a/sdk/objc/native/src/objc_video_track_source.mm +++ b/sdk/objc/native/src/objc_video_track_source.mm @@ -37,7 +37,7 @@ - (void)capturer:(RTC_OBJC_TYPE(RTCVideoCapturer) *)capturer ObjCVideoTrackSource::ObjCVideoTrackSource() : ObjCVideoTrackSource(false) {} ObjCVideoTrackSource::ObjCVideoTrackSource(bool is_screencast) - : AdaptedVideoTrackSource(/* required resolution alignment */ 2), + : AdaptedVideoTrackSource(/* required resolution alignment */ is_screencast? 16 : 2), is_screencast_(is_screencast) {} ObjCVideoTrackSource::ObjCVideoTrackSource(RTCObjCVideoSourceAdapter *adapter) : adapter_(adapter) {