Skip to content

Commit aecf5ce

Browse files
bparrishMinesbparrishMines
authored
Add the ability for ML Kit to create image from bytes (flutter#971)
1 parent c6759fa commit aecf5ce

File tree

12 files changed

+375
-55
lines changed

12 files changed

+375
-55
lines changed

‎packages/firebase_ml_vision/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 0.2.1
2+
3+
* Add capability to create image from bytes.
4+
15
## 0.2.0+2
26

37
* Fix bug with empty text object.

‎packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java‎

Lines changed: 53 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import android.net.Uri;
44
import com.google.firebase.ml.vision.common.FirebaseVisionImage;
5+
import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata;
56
import io.flutter.plugin.common.MethodCall;
67
import io.flutter.plugin.common.MethodChannel;
78
import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
@@ -29,55 +30,74 @@ public static void registerWith(Registrar registrar) {
2930
@Override
3031
public void onMethodCall(MethodCall call, Result result) {
3132
Map<String, Object> options = call.argument("options");
33+
3234
FirebaseVisionImage image;
35+
Map<String, Object> imageData = call.arguments();
36+
try {
37+
image = dataToVisionImage(imageData);
38+
} catch (IOException exception) {
39+
result.error("MLVisionDetectorIOError", exception.getLocalizedMessage(), null);
40+
return;
41+
}
42+
3343
switch (call.method) {
3444
case "BarcodeDetector#detectInImage":
35-
try {
36-
image = filePathToVisionImage((String) call.argument("path"));
37-
BarcodeDetector.instance.handleDetection(image, options, result);
38-
} catch (IOException e) {
39-
result.error("barcodeDetectorIOError", e.getLocalizedMessage(), null);
40-
}
45+
BarcodeDetector.instance.handleDetection(image, options, result);
4146
break;
4247
case "FaceDetector#detectInImage":
43-
try {
44-
image = filePathToVisionImage((String) call.argument("path"));
45-
FaceDetector.instance.handleDetection(image, options, result);
46-
} catch (IOException e) {
47-
result.error("faceDetectorIOError", e.getLocalizedMessage(), null);
48-
}
48+
FaceDetector.instance.handleDetection(image, options, result);
4949
break;
5050
case "LabelDetector#detectInImage":
51-
try {
52-
image = filePathToVisionImage((String) call.argument("path"));
53-
LabelDetector.instance.handleDetection(image, options, result);
54-
} catch (IOException e) {
55-
result.error("labelDetectorIOError", e.getLocalizedMessage(), null);
56-
}
51+
LabelDetector.instance.handleDetection(image, options, result);
5752
break;
5853
case "CloudLabelDetector#detectInImage":
59-
try {
60-
image = filePathToVisionImage((String) call.argument("path"));
61-
CloudLabelDetector.instance.handleDetection(image, options, result);
62-
} catch (IOException e) {
63-
result.error("cloudLabelDetectorIOError", e.getLocalizedMessage(), null);
64-
}
54+
CloudLabelDetector.instance.handleDetection(image, options, result);
6555
break;
6656
case "TextRecognizer#processImage":
67-
try {
68-
image = filePathToVisionImage((String) call.argument("path"));
69-
TextRecognizer.instance.handleDetection(image, options, result);
70-
} catch (IOException e) {
71-
result.error("textRecognizerIOError", e.getLocalizedMessage(), null);
72-
}
57+
TextRecognizer.instance.handleDetection(image, options, result);
7358
break;
7459
default:
7560
result.notImplemented();
7661
}
7762
}
7863

79-
private FirebaseVisionImage filePathToVisionImage(String path) throws IOException {
80-
File file = new File(path);
81-
return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file));
64+
private FirebaseVisionImage dataToVisionImage(Map<String, Object> imageData) throws IOException {
65+
String imageType = (String) imageData.get("type");
66+
67+
switch (imageType) {
68+
case "file":
69+
File file = new File((String) imageData.get("path"));
70+
return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file));
71+
case "bytes":
72+
@SuppressWarnings("unchecked")
73+
Map<String, Object> metadataData = (Map<String, Object>) imageData.get("metadata");
74+
75+
FirebaseVisionImageMetadata metadata =
76+
new FirebaseVisionImageMetadata.Builder()
77+
.setWidth((int) (double) metadataData.get("width"))
78+
.setHeight((int) (double) metadataData.get("height"))
79+
.setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21)
80+
.setRotation(getRotation((int) metadataData.get("rotation")))
81+
.build();
82+
83+
return FirebaseVisionImage.fromByteArray((byte[]) imageData.get("bytes"), metadata);
84+
default:
85+
throw new IllegalArgumentException(String.format("No image type for: %s", imageType));
86+
}
87+
}
88+
89+
private int getRotation(int rotation) {
90+
switch (rotation) {
91+
case 0:
92+
return FirebaseVisionImageMetadata.ROTATION_0;
93+
case 90:
94+
return FirebaseVisionImageMetadata.ROTATION_90;
95+
case 180:
96+
return FirebaseVisionImageMetadata.ROTATION_180;
97+
case 270:
98+
return FirebaseVisionImageMetadata.ROTATION_270;
99+
default:
100+
throw new IllegalArgumentException(String.format("No rotation for: %d", rotation));
101+
}
82102
}
83103
}

‎packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m‎

Lines changed: 70 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ - (instancetype)init {
3636
}
3737

3838
- (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result {
39-
FIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]];
39+
FIRVisionImage *image = [self dataToVisionImage:call.arguments];
4040
NSDictionary *options = call.arguments[@"options"];
4141
if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) {
4242
[BarcodeDetector handleDetection:image options:options result:result];
@@ -53,8 +53,74 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result
5353
}
5454
}
5555

56-
- (FIRVisionImage *)filePathToVisionImage:(NSString *)path {
57-
UIImage *image = [UIImage imageWithContentsOfFile:path];
58-
return [[FIRVisionImage alloc] initWithImage:image];
56+
- (FIRVisionImage *)dataToVisionImage:(NSDictionary *)imageData {
57+
NSString *imageType = imageData[@"type"];
58+
59+
if ([@"file" isEqualToString:imageType]) {
60+
UIImage *image = [UIImage imageWithContentsOfFile:imageData[@"path"]];
61+
return [[FIRVisionImage alloc] initWithImage:image];
62+
} else if ([@"bytes" isEqualToString:imageType]) {
63+
FlutterStandardTypedData *byteData = imageData[@"bytes"];
64+
NSData *imageBytes = byteData.data;
65+
66+
NSDictionary *metadata = imageData[@"metadata"];
67+
NSArray *planeData = metadata[@"planeData"];
68+
size_t planeCount = planeData.count;
69+
70+
size_t widths[planeCount];
71+
size_t heights[planeCount];
72+
size_t bytesPerRows[planeCount];
73+
74+
void *baseAddresses[planeCount];
75+
baseAddresses[0] = (void *)imageBytes.bytes;
76+
77+
size_t lastAddressIndex = 0; // Used to get base address for each plane
78+
for (int i = 0; i < planeCount; i++) {
79+
NSDictionary *plane = planeData[i];
80+
81+
NSNumber *width = plane[@"width"];
82+
NSNumber *height = plane[@"height"];
83+
NSNumber *bytesPerRow = plane[@"bytesPerRow"];
84+
85+
widths[i] = width.unsignedLongValue;
86+
heights[i] = height.unsignedLongValue;
87+
bytesPerRows[i] = bytesPerRow.unsignedLongValue;
88+
89+
if (i > 0) {
90+
size_t addressIndex = lastAddressIndex + heights[i - 1] * bytesPerRows[i - 1];
91+
baseAddresses[i] = (void *)imageBytes.bytes + addressIndex;
92+
lastAddressIndex = addressIndex;
93+
}
94+
}
95+
96+
NSNumber *width = metadata[@"width"];
97+
NSNumber *height = metadata[@"height"];
98+
99+
NSNumber *rawFormat = metadata[@"rawFormat"];
100+
FourCharCode format = FOUR_CHAR_CODE(rawFormat.unsignedIntValue);
101+
102+
CVPixelBufferRef pxbuffer = NULL;
103+
CVPixelBufferCreateWithPlanarBytes(kCFAllocatorDefault, width.unsignedLongValue,
104+
height.unsignedLongValue, format, NULL, imageBytes.length, 2,
105+
baseAddresses, widths, heights, bytesPerRows, NULL, NULL,
106+
NULL, &pxbuffer);
107+
108+
CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pxbuffer];
109+
110+
CIContext *temporaryContext = [CIContext contextWithOptions:nil];
111+
CGImageRef videoImage =
112+
[temporaryContext createCGImage:ciImage
113+
fromRect:CGRectMake(0, 0, CVPixelBufferGetWidth(pxbuffer),
114+
CVPixelBufferGetHeight(pxbuffer))];
115+
116+
UIImage *uiImage = [UIImage imageWithCGImage:videoImage];
117+
CGImageRelease(videoImage);
118+
return [[FIRVisionImage alloc] initWithImage:uiImage];
119+
} else {
120+
NSString *errorReason = [NSString stringWithFormat:@"No image type for: %@", imageType];
121+
@throw [NSException exceptionWithName:NSInvalidArgumentException
122+
reason:errorReason
123+
userInfo:nil];
124+
}
59125
}
60126
@end

‎packages/firebase_ml_vision/lib/firebase_ml_vision.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ library firebase_ml_vision;
77
import 'dart:async';
88
import 'dart:io';
99
import 'dart:math';
10+
import 'dart:typed_data';
11+
import 'dart:ui';
1012

1113
import 'package:flutter/foundation.dart';
1214
import 'package:flutter/services.dart';

‎packages/firebase_ml_vision/lib/src/barcode_detector.dart‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,11 +189,10 @@ class BarcodeDetector extends FirebaseVisionDetector {
189189
final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
190190
'BarcodeDetector#detectInImage',
191191
<String, dynamic>{
192-
'path': visionImage.imageFile.path,
193192
'options': <String, dynamic>{
194193
'barcodeFormats': options.barcodeFormats.value,
195194
},
196-
},
195+
}..addAll(visionImage._serialize()),
197196
);
198197

199198
final List<Barcode> barcodes = <Barcode>[];

‎packages/firebase_ml_vision/lib/src/cloud_detector_options.dart‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class CloudDetectorOptions {
2929
/// The type of model to use for the detection.
3030
final CloudModelType modelType;
3131

32-
Map<String, dynamic> _toMap() => <String, dynamic>{
32+
Map<String, dynamic> _serialize() => <String, dynamic>{
3333
'maxResults': maxResults,
3434
'modelType': _enumToString(modelType),
3535
};

‎packages/firebase_ml_vision/lib/src/face_detector.dart‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,14 @@ class FaceDetector extends FirebaseVisionDetector {
4444
final List<dynamic> reply = await FirebaseVision.channel.invokeMethod(
4545
'FaceDetector#detectInImage',
4646
<String, dynamic>{
47-
'path': visionImage.imageFile.path,
4847
'options': <String, dynamic>{
4948
'enableClassification': options.enableClassification,
5049
'enableLandmarks': options.enableLandmarks,
5150
'enableTracking': options.enableTracking,
5251
'minFaceSize': options.minFaceSize,
5352
'mode': _enumToString(options.mode),
5453
},
55-
},
54+
}..addAll(visionImage._serialize()),
5655
);
5756

5857
final List<Face> faces = <Face>[];

0 commit comments

Comments
 (0)