Skip to content
This repository was archived by the owner on Feb 22, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/camera/camera/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## 0.9.4+10

* iOS performance improvement by moving file writing from the main queue to a background IO queue.

## 0.9.4+9

* iOS performance improvement by moving sample buffer handling from the main queue to a background session queue.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
archiveVersion = 1;
classes = {
};
objectVersion = 46;
objectVersion = 50;
objects = {

/* Begin PBXBuildFile section */
Expand All @@ -22,6 +22,7 @@
97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */; };
E01EE4A82799F3A5008C1950 /* QueueHelperTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E01EE4A72799F3A5008C1950 /* QueueHelperTests.m */; };
E032F250279F5E94009E9028 /* CameraCaptureSessionQueueRaceConditionTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E032F24F279F5E94009E9028 /* CameraCaptureSessionQueueRaceConditionTests.m */; };
E04F108627A87CA600573D0C /* FLTSavePhotoDelegateTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E04F108527A87CA600573D0C /* FLTSavePhotoDelegateTests.m */; };
E0C6E2002770F01A00EA6AA3 /* ThreadSafeMethodChannelTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E0C6E1FD2770F01A00EA6AA3 /* ThreadSafeMethodChannelTests.m */; };
E0C6E2012770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E0C6E1FE2770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m */; };
E0C6E2022770F01A00EA6AA3 /* ThreadSafeEventChannelTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E0C6E1FF2770F01A00EA6AA3 /* ThreadSafeEventChannelTests.m */; };
Expand Down Expand Up @@ -83,6 +84,7 @@
A24F9E418BA48BCC7409B117 /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.release.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig"; sourceTree = "<group>"; };
E01EE4A72799F3A5008C1950 /* QueueHelperTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = QueueHelperTests.m; sourceTree = "<group>"; };
E032F24F279F5E94009E9028 /* CameraCaptureSessionQueueRaceConditionTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CameraCaptureSessionQueueRaceConditionTests.m; sourceTree = "<group>"; };
E04F108527A87CA600573D0C /* FLTSavePhotoDelegateTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FLTSavePhotoDelegateTests.m; sourceTree = "<group>"; };
E0C6E1FD2770F01A00EA6AA3 /* ThreadSafeMethodChannelTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ThreadSafeMethodChannelTests.m; sourceTree = "<group>"; };
E0C6E1FE2770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ThreadSafeTextureRegistryTests.m; sourceTree = "<group>"; };
E0C6E1FF2770F01A00EA6AA3 /* ThreadSafeEventChannelTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ThreadSafeEventChannelTests.m; sourceTree = "<group>"; };
Expand Down Expand Up @@ -125,6 +127,7 @@
E0C6E1FD2770F01A00EA6AA3 /* ThreadSafeMethodChannelTests.m */,
E0C6E1FE2770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m */,
E0F95E4327A36B9200699390 /* SampleBufferQueueTests.m */,
E04F108527A87CA600573D0C /* FLTSavePhotoDelegateTests.m */,
E01EE4A72799F3A5008C1950 /* QueueHelperTests.m */,
E487C85F26D686A10034AC92 /* CameraPreviewPauseTests.m */,
F6EE622E2710A6FC00905E4A /* MockFLTThreadSafeFlutterResult.m */,
Expand Down Expand Up @@ -399,6 +402,7 @@
E0F95E3D27A32AB900699390 /* CameraPropertiesTests.m in Sources */,
03BB766B2665316900CE5A93 /* CameraFocusTests.m in Sources */,
E487C86026D686A10034AC92 /* CameraPreviewPauseTests.m in Sources */,
E04F108627A87CA600573D0C /* FLTSavePhotoDelegateTests.m in Sources */,
F6EE622F2710A6FC00905E4A /* MockFLTThreadSafeFlutterResult.m in Sources */,
334733EA2668111C00DCC49E /* CameraOrientationTests.m in Sources */,
E032F250279F5E94009E9028 /* CameraCaptureSessionQueueRaceConditionTests.m in Sources */,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

@import camera;
@import camera.Test;
@import AVFoundation;
@import XCTest;
#import <OCMock/OCMock.h>

@interface FLTSavePhotoDelegateTests : XCTestCase

@end

@implementation FLTSavePhotoDelegateTests

- (void)testHandlePhotoCaptureResult_mustSendErrorIfFailedToCapture {
NSError *error = [NSError errorWithDomain:@"test" code:0 userInfo:nil];
dispatch_queue_t ioQueue = dispatch_queue_create("test", NULL);
id mockResult = OCMClassMock([FLTThreadSafeFlutterResult class]);
FLTSavePhotoDelegate *delegate = [[FLTSavePhotoDelegate alloc] initWithPath:@"test"
result:mockResult
ioQueue:ioQueue];

[delegate handlePhotoCaptureResultWithError:error
photoDataProvider:^NSData * {
return nil;
}];
OCMVerify([mockResult sendError:error]);
}

- (void)testHandlePhotoCaptureResult_mustSendErrorIfFailedToWrite {
XCTestExpectation *resultExpectation =
[self expectationWithDescription:@"Must send IOError to the result if failed to write file."];
dispatch_queue_t ioQueue = dispatch_queue_create("test", NULL);
id mockResult = OCMClassMock([FLTThreadSafeFlutterResult class]);

NSError *ioError = [NSError errorWithDomain:@"IOError"
code:0
userInfo:@{NSLocalizedDescriptionKey : @"Localized IO Error"}];

OCMStub([mockResult sendErrorWithCode:@"IOError"
message:@"Unable to write file"
details:ioError.localizedDescription])
.andDo(^(NSInvocation *invocation) {
[resultExpectation fulfill];
});
FLTSavePhotoDelegate *delegate = [[FLTSavePhotoDelegate alloc] initWithPath:@"test"
result:mockResult
ioQueue:ioQueue];

// We can't use OCMClassMock for NSData because some XCTest APIs uses NSData (e.g.
// `XCTRunnerIDESession::logDebugMessage:`) on a private queue.
id mockData = OCMPartialMock([NSData data]);
OCMStub([mockData writeToFile:OCMOCK_ANY
options:NSDataWritingAtomic
error:[OCMArg setTo:ioError]])
.andReturn(NO);
[delegate handlePhotoCaptureResultWithError:nil
photoDataProvider:^NSData * {
return mockData;
}];
[self waitForExpectationsWithTimeout:1 handler:nil];
}

- (void)testHandlePhotoCaptureResult_mustSendSuccessIfSuccessToWrite {
XCTestExpectation *resultExpectation = [self
expectationWithDescription:@"Must send file path to the result if success to write file."];

dispatch_queue_t ioQueue = dispatch_queue_create("test", NULL);
id mockResult = OCMClassMock([FLTThreadSafeFlutterResult class]);
FLTSavePhotoDelegate *delegate = [[FLTSavePhotoDelegate alloc] initWithPath:@"test"
result:mockResult
ioQueue:ioQueue];
OCMStub([mockResult sendSuccessWithData:delegate.path]).andDo(^(NSInvocation *invocation) {
[resultExpectation fulfill];
});

// We can't use OCMClassMock for NSData because some XCTest APIs uses NSData (e.g.
// `XCTRunnerIDESession::logDebugMessage:`) on a private queue.
id mockData = OCMPartialMock([NSData data]);
OCMStub([mockData writeToFile:OCMOCK_ANY options:NSDataWritingAtomic error:[OCMArg setTo:nil]])
.andReturn(YES);

[delegate handlePhotoCaptureResultWithError:nil
photoDataProvider:^NSData * {
return mockData;
}];
[self waitForExpectationsWithTimeout:1 handler:nil];
}

- (void)testHandlePhotoCaptureResult_bothProvideDataAndSaveFileMustRunOnIOQueue {
XCTestExpectation *dataProviderQueueExpectation =
[self expectationWithDescription:@"Data provider must run on io queue."];
XCTestExpectation *writeFileQueueExpectation =
[self expectationWithDescription:@"File writing must run on io queue"];
XCTestExpectation *resultExpectation = [self
expectationWithDescription:@"Must send file path to the result if success to write file."];

dispatch_queue_t ioQueue = dispatch_queue_create("test", NULL);
const char *ioQueueSpecific = "io_queue_specific";
dispatch_queue_set_specific(ioQueue, ioQueueSpecific, (void *)ioQueueSpecific, NULL);
id mockResult = OCMClassMock([FLTThreadSafeFlutterResult class]);
OCMStub([mockResult sendSuccessWithData:OCMOCK_ANY]).andDo(^(NSInvocation *invocation) {
[resultExpectation fulfill];
});

// We can't use OCMClassMock for NSData because some XCTest APIs uses NSData (e.g.
// `XCTRunnerIDESession::logDebugMessage:`) on a private queue.
id mockData = OCMPartialMock([NSData data]);
OCMStub([mockData writeToFile:OCMOCK_ANY options:NSDataWritingAtomic error:[OCMArg setTo:nil]])
.andDo(^(NSInvocation *invocation) {
if (dispatch_get_specific(ioQueueSpecific)) {
[writeFileQueueExpectation fulfill];
}
})
.andReturn(YES);

FLTSavePhotoDelegate *delegate = [[FLTSavePhotoDelegate alloc] initWithPath:@"test"
result:mockResult
ioQueue:ioQueue];
[delegate handlePhotoCaptureResultWithError:nil
photoDataProvider:^NSData * {
if (dispatch_get_specific(ioQueueSpecific)) {
[dataProviderQueueExpectation fulfill];
}
return mockData;
}];

[self waitForExpectationsWithTimeout:1 handler:nil];
}

@end
6 changes: 6 additions & 0 deletions packages/camera/camera/ios/Classes/CameraPlugin.modulemap
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,11 @@ framework module camera {
header "CameraProperties.h"
header "FLTCam.h"
header "FLTCam_Test.h"
header "FLTSavePhotoDelegate_Test.h"
header "FLTThreadSafeEventChannel.h"
header "FLTThreadSafeFlutterResult.h"
header "FLTThreadSafeMethodChannel.h"
header "FLTThreadSafeTextureRegistry.h"
header "QueueHelper.h"
}
}
80 changes: 11 additions & 69 deletions packages/camera/camera/ios/Classes/FLTCam.m
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

#import "FLTCam.h"
#import "FLTCam_Test.h"
#import "FLTSavePhotoDelegate.h"

@import CoreMotion;
#import <libkern/OSAtomic.h>
Expand Down Expand Up @@ -41,71 +42,6 @@ - (FlutterError *_Nullable)onListenWithArguments:(id _Nullable)arguments
}
@end

@interface FLTSavePhotoDelegate : NSObject <AVCapturePhotoCaptureDelegate>
@property(readonly, nonatomic) NSString *path;
@property(readonly, nonatomic) FLTThreadSafeFlutterResult *result;
@end

@implementation FLTSavePhotoDelegate {
/// Used to keep the delegate alive until didFinishProcessingPhotoSampleBuffer.
FLTSavePhotoDelegate *selfReference;
}

- initWithPath:(NSString *)path result:(FLTThreadSafeFlutterResult *)result {
self = [super init];
NSAssert(self, @"super init cannot be nil");
_path = path;
selfReference = self;
_result = result;
return self;
}

- (void)captureOutput:(AVCapturePhotoOutput *)output
didFinishProcessingPhotoSampleBuffer:(CMSampleBufferRef)photoSampleBuffer
previewPhotoSampleBuffer:(CMSampleBufferRef)previewPhotoSampleBuffer
resolvedSettings:(AVCaptureResolvedPhotoSettings *)resolvedSettings
bracketSettings:(AVCaptureBracketedStillImageSettings *)bracketSettings
error:(NSError *)error API_AVAILABLE(ios(10)) {
selfReference = nil;
if (error) {
[_result sendError:error];
return;
}

NSData *data = [AVCapturePhotoOutput
JPEGPhotoDataRepresentationForJPEGSampleBuffer:photoSampleBuffer
previewPhotoSampleBuffer:previewPhotoSampleBuffer];

// TODO(sigurdm): Consider writing file asynchronously.
bool success = [data writeToFile:_path atomically:YES];

if (!success) {
[_result sendErrorWithCode:@"IOError" message:@"Unable to write file" details:nil];
return;
}
[_result sendSuccessWithData:_path];
}

- (void)captureOutput:(AVCapturePhotoOutput *)output
didFinishProcessingPhoto:(AVCapturePhoto *)photo
error:(NSError *)error API_AVAILABLE(ios(11.0)) {
selfReference = nil;
if (error) {
[_result sendError:error];
return;
}

NSData *photoData = [photo fileDataRepresentation];

bool success = [photoData writeToFile:_path atomically:YES];
if (!success) {
[_result sendErrorWithCode:@"IOError" message:@"Unable to write file" details:nil];
return;
}
[_result sendSuccessWithData:_path];
}
@end

@interface FLTCam () <AVCaptureVideoDataOutputSampleBufferDelegate,
AVCaptureAudioDataOutputSampleBufferDelegate>

Expand Down Expand Up @@ -138,8 +74,11 @@ @interface FLTCam () <AVCaptureVideoDataOutputSampleBufferDelegate,
@property(assign, nonatomic) CMTime audioTimeOffset;
@property(nonatomic) CMMotionManager *motionManager;
@property AVAssetWriterInputPixelBufferAdaptor *videoAdaptor;
// All FLTCam's state access and capture session related operations should be on run on this queue.
/// All FLTCam's state access and capture session related operations should be on run on this queue.
@property(strong, nonatomic) dispatch_queue_t captureSessionQueue;
/// The queue on which captured photos (not videos) are wrote to disk.
/// Videos are wrote to disk by `videoAdaptor` on an internal queue managed by AVFoundation.
@property(strong, nonatomic) dispatch_queue_t photoIOQueue;
@property(assign, nonatomic) UIDeviceOrientation deviceOrientation;
@end

Expand All @@ -162,6 +101,7 @@ - (instancetype)initWithCameraName:(NSString *)cameraName
}
_enableAudio = enableAudio;
_captureSessionQueue = captureSessionQueue;
_photoIOQueue = dispatch_queue_create("io.flutter.camera.photoIOQueue", NULL);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

captureSessionQueue was injected because it's also used in CameraPlugin.m

_captureSession = [[AVCaptureSession alloc] init];
_captureDevice = [AVCaptureDevice deviceWithUniqueID:cameraName];
_flashMode = _captureDevice.hasFlash ? FLTFlashModeAuto : FLTFlashModeOff;
Expand Down Expand Up @@ -280,9 +220,11 @@ - (void)captureToFile:(FLTThreadSafeFlutterResult *)result API_AVAILABLE(ios(10)
return;
}

[_capturePhotoOutput capturePhotoWithSettings:settings
delegate:[[FLTSavePhotoDelegate alloc] initWithPath:path
result:result]];
[_capturePhotoOutput
capturePhotoWithSettings:settings
delegate:[[FLTSavePhotoDelegate alloc] initWithPath:path
result:result
ioQueue:self.photoIOQueue]];
}

- (AVCaptureVideoOrientation)getVideoOrientationForDeviceOrientation:
Expand Down
37 changes: 37 additions & 0 deletions packages/camera/camera/ios/Classes/FLTSavePhotoDelegate.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is now exposed in the plugin API, I don't think we want that. I think keep most of this in the implementation and only expose FLTSavePhotoDelegate_Test.h in the test module.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I didn't add it to the umbrella header so it's not exposed outside of this package.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also removed thread safe wrappers from the umbrella header

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh right right I forgot this uses a real umbrella header and not one generated by CocoaPods.

I think you're still exposing it in the test module since you're #importing it in the _Test header.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep

Copy link
Contributor Author

@hellohuanlin hellohuanlin Feb 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The original header and the test header are both exposed for unit test only. I think this is what we want.

// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

@import AVFoundation;
@import Foundation;
@import Flutter;

#import "FLTThreadSafeFlutterResult.h"

NS_ASSUME_NONNULL_BEGIN

/**
Delegate object that handles photo capture results.
*/
@interface FLTSavePhotoDelegate : NSObject <AVCapturePhotoCaptureDelegate>
/// The file path for the captured photo.
@property(readonly, nonatomic) NSString *path;
/// The thread safe flutter result wrapper to report the result.
@property(readonly, nonatomic) FLTThreadSafeFlutterResult *result;
/// The queue on which captured photos are wrote to disk.
@property(strong, nonatomic) dispatch_queue_t ioQueue;
/// Used to keep the delegate alive until didFinishProcessingPhotoSampleBuffer.
@property(strong, nonatomic, nullable) FLTSavePhotoDelegate *selfReference;

/**
* Initialize a photo capture delegate.
* @param path the path for captured photo file.
* @param result the thread safe flutter result wrapper to report the result.
* @param ioQueue the queue on which captured photos are wrote to disk.
*/
- (instancetype)initWithPath:(NSString *)path
result:(FLTThreadSafeFlutterResult *)result
ioQueue:(dispatch_queue_t)ioQueue;
@end

NS_ASSUME_NONNULL_END
Loading