From 39e761ba9df9d5b44d3d289aad65f2f431116ab1 Mon Sep 17 00:00:00 2001 From: Jeronimo Valli Date: Thu, 22 Jun 2023 21:03:51 +0200 Subject: [PATCH 1/3] VIDCS-974: React Native- IOS: App crash --- ios/OpenTokReactNative/OTScreenCapture.h | 11 - ios/OpenTokReactNative/OTScreenCapture.m | 358 ++++++----------------- 2 files changed, 91 insertions(+), 278 deletions(-) diff --git a/ios/OpenTokReactNative/OTScreenCapture.h b/ios/OpenTokReactNative/OTScreenCapture.h index 0cc28476..665e6612 100644 --- a/ios/OpenTokReactNative/OTScreenCapture.h +++ b/ios/OpenTokReactNative/OTScreenCapture.h @@ -10,13 +10,6 @@ @protocol OTVideoCapture; -// defines for image scaling -// From https://bugs.chromium.org/p/webrtc/issues/detail?id=4643#c7 : -// Don't send any image larger than 1280px on either edge. Additionally, don't -// send any image with dimensions %16 != 0 -#define MAX_EDGE_SIZE_LIMIT 1280.0f -#define EDGE_DIMENSION_COMMON_FACTOR 16.0f - /** * Periodically sends video frames to an OpenTok Publisher by rendering the * CALayer for a UIView. @@ -30,9 +23,5 @@ */ - (instancetype)initWithView:(UIView*)view; -// private: declared here for testing scaling & padding function -+ (void)dimensionsForInputSize:(CGSize)input - containerSize:(CGSize*)destContainerSize - drawRect:(CGRect*)destDrawRect; @end diff --git a/ios/OpenTokReactNative/OTScreenCapture.m b/ios/OpenTokReactNative/OTScreenCapture.m index a8e804e0..db3c4704 100644 --- a/ios/OpenTokReactNative/OTScreenCapture.m +++ b/ios/OpenTokReactNative/OTScreenCapture.m @@ -7,21 +7,26 @@ #include #include +#import #import "OTScreenCapture.h" @implementation OTScreenCapture { - CMTime _minFrameDuration; dispatch_queue_t _queue; - dispatch_source_t _timer; CVPixelBufferRef _pixelBuffer; BOOL _capturing; OTVideoFrame* _videoFrame; UIView* _view; + CGFloat _screenScale; + CGContextRef _bitmapContext; + + CADisplayLink *_displayLink; + dispatch_semaphore_t _capturingSemaphore; } @synthesize videoCaptureConsumer; +@synthesize videoContentHint; #pragma mark - Class Lifecycle. @@ -30,16 +35,13 @@ - (instancetype)initWithView:(UIView *)view self = [super init]; if (self) { _view = view; - // Recommend sending 5 frames per second: Allows for higher image - // quality per frame - _minFrameDuration = CMTimeMake(1, 5); _queue = dispatch_queue_create("SCREEN_CAPTURE", NULL); - - OTVideoFormat *format = [[OTVideoFormat alloc] init]; - [format setPixelFormat:OTPixelFormatARGB]; - - _videoFrame = [[OTVideoFrame alloc] initWithFormat:format]; - + _screenScale = [[UIScreen mainScreen] scale]; + _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(captureView)]; + _displayLink.preferredFramesPerSecond = 30.0; + _capturingSemaphore = dispatch_semaphore_create(1); + [self createPixelBuffer]; + [self createCGContextFromPixelBuffer]; } return self; } @@ -47,100 +49,117 @@ - (instancetype)initWithView:(UIView *)view - (void)dealloc { [self stopCapture]; - CVPixelBufferRelease(_pixelBuffer); + if(_bitmapContext) + CGContextRelease(_bitmapContext); + if(_pixelBuffer) + CVPixelBufferRelease(_pixelBuffer); } #pragma mark - Private Methods -/** - * Make sure receiving video frame container is setup for this image. - */ -- (void)checkImageSize:(CGImageRef)image { - CGFloat width = CGImageGetWidth(image); - CGFloat height = CGImageGetHeight(image); - - if (_videoFrame.format.imageHeight == height && - _videoFrame.format.imageWidth == width) - { - // don't rock the boat. if nothing has changed, don't update anything. +- (CMTime)getTimeStamp { + static mach_timebase_info_data_t time_info; + uint64_t time_stamp = 0; + if (time_info.denom == 0) { + (void) mach_timebase_info(&time_info); + } + time_stamp = mach_absolute_time(); + time_stamp *= time_info.numer; + time_stamp /= time_info.denom; + CMTime time = CMTimeMake(time_stamp, 1000); + return time; +} + +-(void)captureView +{ + if (!(_capturing && self.videoCaptureConsumer)) { return; } - - [_videoFrame.format.bytesPerRow removeAllObjects]; - [_videoFrame.format.bytesPerRow addObject:@(width * 4)]; - [_videoFrame.format setImageHeight:height]; - [_videoFrame.format setImageWidth:width]; - - CGSize frameSize = CGSizeMake(width, height); - NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys: - @NO, - kCVPixelBufferCGImageCompatibilityKey, - @NO, - kCVPixelBufferCGBitmapContextCompatibilityKey, - nil]; - - if (NULL != _pixelBuffer) { - CVPixelBufferRelease(_pixelBuffer); + + // Wait until consumeImageBuffer is done. + if (dispatch_semaphore_wait(_capturingSemaphore, DISPATCH_TIME_NOW) != 0) { + return; } - CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, - frameSize.width, - frameSize.height, - kCVPixelFormatType_32ARGB, - (__bridge CFDictionaryRef)(options), - &_pixelBuffer); + dispatch_async(dispatch_get_main_queue(), ^{ + [self.view.layer renderInContext:self->_bitmapContext]; + // Don't block the UI thread + dispatch_async(self->_queue, ^{ + CMTime time = [self getTimeStamp]; + [self.videoCaptureConsumer consumeImageBuffer:self->_pixelBuffer + orientation:OTVideoOrientationUp + timestamp:time + metadata:nil]; + // Signal for more frames + dispatch_semaphore_signal(self->_capturingSemaphore); + }); + }); +} + +- (void)createCGContextFromPixelBuffer { + CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); + CVPixelBufferLockBaseAddress(_pixelBuffer, 0); - NSParameterAssert(status == kCVReturnSuccess && _pixelBuffer != NULL); + _bitmapContext = NULL; + _bitmapContext = CGBitmapContextCreate(CVPixelBufferGetBaseAddress(_pixelBuffer), + CVPixelBufferGetWidth(_pixelBuffer), + CVPixelBufferGetHeight(_pixelBuffer), + 8, CVPixelBufferGetBytesPerRow(_pixelBuffer), rgbColorSpace, + kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst + ); + CGContextTranslateCTM(_bitmapContext, 0.0f, self.view.frame.size.height); + CGContextScaleCTM(_bitmapContext, self.view.layer.contentsScale, -self.view.layer.contentsScale); + CVPixelBufferUnlockBaseAddress(_pixelBuffer, 0); + CFRelease(rgbColorSpace); +} +- (void)createPixelBuffer { + + CFDictionaryRef ioSurfaceProps = CFDictionaryCreate( kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks ); + + NSDictionary *bufferAttributes = @{(id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA), + (id)kCVPixelBufferCGBitmapContextCompatibilityKey : @YES, + (id)kCVPixelBufferWidthKey : @(self.view.frame.size.width * _screenScale), + (id)kCVPixelBufferHeightKey : @(self.view.frame.size.height * _screenScale), + (id)kCVPixelBufferBytesPerRowAlignmentKey : + @(self.view.frame.size.width * _screenScale * 4), + (id)kCVPixelBufferIOSurfacePropertiesKey : (__bridge id)ioSurfaceProps + }; + CVPixelBufferCreate(kCFAllocatorDefault, + self.view.frame.size.width, + self.view.frame.size.height, + kCVPixelFormatType_32ARGB, + (__bridge CFDictionaryRef)(bufferAttributes), + &_pixelBuffer); + CFRelease(ioSurfaceProps); } #pragma mark - Capture lifecycle /** - * Allocate capture resources; in this case we're just setting up a timer and + * Allocate capture resources; in this case we're just setting up a timer and * block to execute periodically to send video frames. */ + - (void)initCapture { - __unsafe_unretained OTScreenCapture* _self = self; - _timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, _queue); - dispatch_source_set_timer(_timer, dispatch_walltime(NULL, 0), - 100ull * NSEC_PER_MSEC, 100ull * NSEC_PER_MSEC); - - dispatch_source_set_event_handler(_timer, ^{ - @autoreleasepool { - __block UIImage* screen = [_self screenshot]; - CGImageRef paddedScreen = [self resizeAndPadImage:screen]; - [_self consumeFrame:paddedScreen]; - } - }); } - (void)releaseCapture { - _timer = nil; + [_displayLink invalidate]; } - (int32_t)startCapture { _capturing = YES; - - if (_timer) { - dispatch_resume(_timer); - } - + [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes]; return 0; } - (int32_t)stopCapture { _capturing = NO; - - dispatch_sync(_queue, ^{ - if (self->_timer) { - dispatch_source_cancel(self->_timer); - } - }); - + [_displayLink invalidate]; return 0; } @@ -149,205 +168,10 @@ - (BOOL)isCaptureStarted return _capturing; } -#pragma mark - Screen capture implementation - -- (CVPixelBufferRef)pixelBufferFromCGImage:(CGImageRef)image -{ - CGFloat width = CGImageGetWidth(image); - CGFloat height = CGImageGetHeight(image); - CGSize frameSize = CGSizeMake(width, height); - CVPixelBufferLockBaseAddress(_pixelBuffer, 0); - void *pxdata = CVPixelBufferGetBaseAddress(_pixelBuffer); - - CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); - CGContextRef context = - CGBitmapContextCreate(pxdata, - frameSize.width, - frameSize.height, - 8, - CVPixelBufferGetBytesPerRow(_pixelBuffer), - rgbColorSpace, - kCGImageAlphaPremultipliedFirst | - kCGBitmapByteOrder32Little); - - - CGContextDrawImage(context, CGRectMake(0, 0, width, height), image); - CGColorSpaceRelease(rgbColorSpace); - CGContextRelease(context); - - CVPixelBufferUnlockBaseAddress(_pixelBuffer, 0); - - return _pixelBuffer; -} - - (int32_t)captureSettings:(OTVideoFormat*)videoFormat { videoFormat.pixelFormat = OTPixelFormatARGB; return 0; } -+ (void)dimensionsForInputSize:(CGSize)input - containerSize:(CGSize*)destContainerSize - drawRect:(CGRect*)destDrawRect -{ - CGFloat sourceWidth = input.width; - CGFloat sourceHeight = input.height; - double sourceAspectRatio = sourceWidth / sourceHeight; - - CGFloat destContainerWidth = sourceWidth; - CGFloat destContainerHeight = sourceHeight; - CGFloat destImageWidth = sourceWidth; - CGFloat destImageHeight = sourceHeight; - - // if image is wider than tall and width breaks edge size limit - if (MAX_EDGE_SIZE_LIMIT < sourceWidth && sourceAspectRatio >= 1.0) { - destContainerWidth = MAX_EDGE_SIZE_LIMIT; - destContainerHeight = destContainerWidth / sourceAspectRatio; - if (0 != fmod(destContainerHeight, EDGE_DIMENSION_COMMON_FACTOR)) { - // add padding to make height % 16 == 0 - destContainerHeight += - (EDGE_DIMENSION_COMMON_FACTOR - fmod(destContainerHeight, - EDGE_DIMENSION_COMMON_FACTOR)); - } - destImageWidth = destContainerWidth; - destImageHeight = destContainerWidth / sourceAspectRatio; - } - - // if image is taller than wide and height breaks edge size limit - if (MAX_EDGE_SIZE_LIMIT < destContainerHeight && sourceAspectRatio <= 1.0) { - destContainerHeight = MAX_EDGE_SIZE_LIMIT; - destContainerWidth = destContainerHeight * sourceAspectRatio; - if (0 != fmod(destContainerWidth, EDGE_DIMENSION_COMMON_FACTOR)) { - // add padding to make width % 16 == 0 - destContainerWidth += - (EDGE_DIMENSION_COMMON_FACTOR - fmod(destContainerWidth, - EDGE_DIMENSION_COMMON_FACTOR)); - } - destImageHeight = destContainerHeight; - destImageWidth = destContainerHeight * sourceAspectRatio; - } - - // ensure the dimensions of the resulting container are safe - if (fmod(destContainerWidth, EDGE_DIMENSION_COMMON_FACTOR) != 0) { - double remainder = fmod(destContainerWidth, - EDGE_DIMENSION_COMMON_FACTOR); - // increase the edge size only if doing so does not break the edge limit - if (destContainerWidth + (EDGE_DIMENSION_COMMON_FACTOR - remainder) > - MAX_EDGE_SIZE_LIMIT) - { - destContainerWidth -= remainder; - } else { - destContainerWidth += EDGE_DIMENSION_COMMON_FACTOR - remainder; - } - } - // ensure the dimensions of the resulting container are safe - if (fmod(destContainerHeight, EDGE_DIMENSION_COMMON_FACTOR) != 0) { - double remainder = fmod(destContainerHeight, - EDGE_DIMENSION_COMMON_FACTOR); - // increase the edge size only if doing so does not break the edge limit - if (destContainerHeight + (EDGE_DIMENSION_COMMON_FACTOR - remainder) > - MAX_EDGE_SIZE_LIMIT) - { - destContainerHeight -= remainder; - } else { - destContainerHeight += EDGE_DIMENSION_COMMON_FACTOR - remainder; - } - } - - destContainerSize->width = destContainerWidth; - destContainerSize->height = destContainerHeight; - - // scale and recenter source image to fit in destination container - if (sourceAspectRatio > 1.0) { - destDrawRect->origin.x = 0; - destDrawRect->origin.y = - (destContainerHeight - destImageHeight) / 2; - destDrawRect->size.width = destContainerWidth; - destDrawRect->size.height = - destContainerWidth / sourceAspectRatio; - } else { - destDrawRect->origin.x = - (destContainerWidth - destImageWidth) / 2; - destDrawRect->origin.y = 0; - destDrawRect->size.height = destContainerHeight; - destDrawRect->size.width = - destContainerHeight * sourceAspectRatio; - } - -} - -- (CGImageRef)resizeAndPadImage:(UIImage*)sourceUIImage { - CGImageRef sourceCGImage = [sourceUIImage CGImage]; - CGFloat sourceWidth = CGImageGetWidth(sourceCGImage); - CGFloat sourceHeight = CGImageGetHeight(sourceCGImage); - CGSize sourceSize = CGSizeMake(sourceWidth, sourceHeight); - CGSize destContainerSize = CGSizeZero; - CGRect destRectForSourceImage = CGRectZero; - - [OTScreenCapture dimensionsForInputSize:sourceSize - containerSize:&destContainerSize - drawRect:&destRectForSourceImage]; - - UIGraphicsBeginImageContextWithOptions(destContainerSize, NO, 1.0); - CGContextRef context = UIGraphicsGetCurrentContext(); - - // flip source image to match destination coordinate system - CGContextScaleCTM(context, 1.0, -1.0); - CGContextTranslateCTM(context, 0, -destRectForSourceImage.size.height); - CGContextDrawImage(context, destRectForSourceImage, sourceCGImage); - - // Clean up and get the new image. - UIImage *newImage = UIGraphicsGetImageFromCurrentImageContext(); - UIGraphicsEndImageContext(); - - return [newImage CGImage]; -} - -- (UIImage *)screenshot -{ - UIGraphicsBeginImageContextWithOptions(self.view.bounds.size, NO, 0.0); - [self.view drawViewHierarchyInRect:self.view.bounds afterScreenUpdates:NO]; - UIImage *image = UIGraphicsGetImageFromCurrentImageContext(); - UIGraphicsEndImageContext(); - return image; -} - -- (void) consumeFrame:(CGImageRef)frame { - - [self checkImageSize:frame]; - - static mach_timebase_info_data_t time_info; - uint64_t time_stamp = 0; - - if (!(_capturing && self.videoCaptureConsumer)) { - return; - } - - if (time_info.denom == 0) { - (void) mach_timebase_info(&time_info); - } - - time_stamp = mach_absolute_time(); - time_stamp *= time_info.numer; - time_stamp /= time_info.denom; - - CMTime time = CMTimeMake(time_stamp, 1000); - CVImageBufferRef ref = [self pixelBufferFromCGImage:frame]; - - CVPixelBufferLockBaseAddress(ref, 0); - - _videoFrame.timestamp = time; - _videoFrame.format.estimatedFramesPerSecond = - _minFrameDuration.timescale / _minFrameDuration.value; - _videoFrame.format.estimatedCaptureDelay = 100; - _videoFrame.orientation = OTVideoOrientationUp; - - [_videoFrame clearPlanes]; - [_videoFrame.planes addPointer:CVPixelBufferGetBaseAddress(ref)]; - [self.videoCaptureConsumer consumeFrame:_videoFrame]; - - CVPixelBufferUnlockBaseAddress(ref, 0); -} - - @end From fc0ad60620284ebf615447dd40a22c18a3f93c9a Mon Sep 17 00:00:00 2001 From: Jeronimo Valli Date: Mon, 26 Jun 2023 09:51:46 -0300 Subject: [PATCH 2/3] code review updates --- ios/OpenTokReactNative/OTScreenCapture.m | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ios/OpenTokReactNative/OTScreenCapture.m b/ios/OpenTokReactNative/OTScreenCapture.m index db3c4704..1e363a76 100644 --- a/ios/OpenTokReactNative/OTScreenCapture.m +++ b/ios/OpenTokReactNative/OTScreenCapture.m @@ -99,8 +99,7 @@ -(void)captureView - (void)createCGContextFromPixelBuffer { CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); CVPixelBufferLockBaseAddress(_pixelBuffer, 0); - - _bitmapContext = NULL; + _bitmapContext = CGBitmapContextCreate(CVPixelBufferGetBaseAddress(_pixelBuffer), CVPixelBufferGetWidth(_pixelBuffer), CVPixelBufferGetHeight(_pixelBuffer), @@ -136,11 +135,6 @@ - (void)createPixelBuffer { #pragma mark - Capture lifecycle -/** - * Allocate capture resources; in this case we're just setting up a timer and - * block to execute periodically to send video frames. - */ - - (void)initCapture { } From 0a174d2af50d207cb11e20dca981eec97a5e8ded Mon Sep 17 00:00:00 2001 From: Jeff Swartz Date: Wed, 5 Jul 2023 14:59:40 -0700 Subject: [PATCH 3/3] Rev version to 2.25.2 --- CHANGELOG.md | 4 ++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdeaa874..15818de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 2.25.2 (July 5 2023) + +- [Fix]: Fix crash on iOS when publishing a screen-sharing stream. + # 2.25.1 (June 27 2023) - [Fix]: Fix camera lifecycle on Android. - issue #645 diff --git a/package-lock.json b/package-lock.json index b15fb4e3..befd45ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "opentok-react-native", - "version": "2.25.1", + "version": "2.25.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "opentok-react-native", - "version": "2.25.1", + "version": "2.25.2", "license": "MIT", "dependencies": { "axios": "^0.21.1", diff --git a/package.json b/package.json index 5da85cf1..1e5bb467 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opentok-react-native", - "version": "2.25.1", + "version": "2.25.2", "description": "React Native components for OpenTok iOS and Android SDKs", "main": "src/index.js", "homepage": "https://www.tokbox.com",