Skip to content

fix: Add support for YUV_420_888 Image format. #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
+178 −59
Open

fix: Add support for YUV_420_888 Image format. #732

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 35 additions & 12 deletions packages/example/lib/vision_detector_views/camera_view.dart
View file
Original file line number Diff line number Diff line change
@@ -358,29 +358,52 @@ class _CameraViewState extends State<CameraView> {

// get image format
final format = InputImageFormatValue.fromRawValue(image.format.raw);
// validate format depending on platform
// only supported formats:
// * nv21 for Android
// * bgra8888 for iOS
if (format == null ||
(Platform.isAndroid && format != InputImageFormat.nv21) ||
if (format == null) {
print('could not find format from raw value: $image.format.raw');
return null;
}
// Validate format depending on platform
final androidSupportedFormats = [
InputImageFormat.nv21,
InputImageFormat.yv12,
InputImageFormat.yuv_420_888
];
if ((Platform.isAndroid && !androidSupportedFormats.contains(format)) ||
(Platform.isIOS && format != InputImageFormat.bgra8888)) {
print('image format is not supported: $format');
return null;
}

// since format is constraint to nv21 or bgra8888, both only have one plane
if (image.planes.length != 1) return null;
final plane = image.planes.first;
// Compile a flat list of all image data. For image formats with multiple planes,
// takes some copying.
final Uint8List bytes = image.planes.length == 1
? image.planes.first.bytes
: _concatenatePlanes(image);

// compose InputImage using bytes
return InputImage.fromBytes(
bytes: plane.bytes,
bytes: bytes,
metadata: InputImageMetadata(
size: Size(image.width.toDouble(), image.height.toDouble()),
rotation: rotation, // used only in Android
format: format, // used only in iOS
bytesPerRow: plane.bytesPerRow, // used only in iOS
format: format,
bytesPerRow: image.planes.first.bytesPerRow, // used only in iOS
),
);
}

Uint8List _concatenatePlanes(CameraImage image) {
int length = 0;
for (final Plane p in image.planes) {
length += p.bytes.length;
}

final Uint8List bytes = Uint8List(length);
int offset = 0;
for (final Plane p in image.planes) {
bytes.setRange(offset, offset + p.bytes.length, p.bytes);
offset += p.bytes.length;
}
return bytes;
}
}
20 changes: 15 additions & 5 deletions ...code_scanning/android/src/main/java/com/google_mlkit_barcode_scanning/BarcodeScanner.java
View file
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
import io.flutter.plugin.common.MethodChannel;

public class BarcodeScanner implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startBarcodeScanner";
private static final String CLOSE = "vision#closeBarcodeScanner";

@@ -67,8 +68,11 @@ private com.google.mlkit.vision.barcode.BarcodeScanner initialize(MethodCall cal

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null) return;
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
com.google.mlkit.vision.barcode.BarcodeScanner barcodeScanner = instances.get(id);
@@ -191,7 +195,9 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
}
result.success(barcodeList);
})
.addOnFailureListener(e -> result.error("BarcodeDetectorError", e.toString(), null));
.addOnFailureListener(e -> result.error("BarcodeDetectorError", e.toString(), e))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());
}

private void addPoints(Point[] cornerPoints, List<Map<String, Integer>> points) {
@@ -205,7 +211,9 @@ private void addPoints(Point[] cornerPoints, List<Map<String, Integer>> points)

private Map<String, Integer> getBoundingPoints(@Nullable Rect rect) {
Map<String, Integer> frame = new HashMap<>();
if (rect == null) return frame;
if (rect == null) {
return frame;
}
frame.put("left", rect.left);
frame.put("right", rect.right);
frame.put("top", rect.top);
@@ -216,7 +224,9 @@ private Map<String, Integer> getBoundingPoints(@Nullable Rect rect) {
private void closeDetector(MethodCall call) {
String id = call.argument("id");
com.google.mlkit.vision.barcode.BarcodeScanner barcodeScanner = instances.get(id);
if (barcodeScanner == null) return;
if (barcodeScanner == null) {
return;
}
barcodeScanner.close();
instances.remove(id);
}
50 changes: 46 additions & 4 deletions ...gle_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java
View file
Original file line number Diff line number Diff line change
@@ -2,22 +2,30 @@

import android.content.Context;
import android.graphics.ImageFormat;
import android.graphics.SurfaceTexture;
import android.media.Image;
import android.media.ImageWriter;
import android.net.Uri;
import android.util.Log;
import android.view.Surface;

import com.google.mlkit.vision.common.InputImage;

import java.io.File;
import java.io.IOException;
import java.lang.AutoCloseable;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.Objects;

import io.flutter.plugin.common.MethodChannel;

public class InputImageConverter {
public class InputImageConverter implements AutoCloseable {

ImageWriter writer;

//Returns an [InputImage] from the image data received
public static InputImage getInputImageFromData(Map<String, Object> imageData,
public InputImage getInputImageFromData(Map<String, Object> imageData,
Context context,
MethodChannel.Result result) {
//Differentiates whether the image data is a path for a image file, contains image data in form of bytes, or a bitmap
@@ -116,9 +124,36 @@ public static InputImage getInputImageFromData(Map<String, Object> imageData,
rotationDegrees,
imageFormat);
}
if (imageFormat == ImageFormat.YUV_420_888) {
// This image format is only supported in InputImage.fromMediaImage, which requires to transform the data to the right java type.
// TODO: Consider reusing the same Surface across multiple calls to save on allocations.
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reusing this ImageWriter across multiple calls, thus avoiding the allocation, would make this likely more performant. Could any of the reviewers take a look?

Is it something that could be left as a TODO and fixed later?

writer = new ImageWriter.Builder(new Surface(new SurfaceTexture(true)))
.setWidthAndHeight(width, height)
.setImageFormat(imageFormat)
.build();
Image image = writer.dequeueInputImage();
if (image == null) {
result.error("InputImageConverterError", "failed to allocate space for input image", null);
return null;
}
// Deconstruct individual planes again from flattened array.
Image.Plane[] planes = image.getPlanes();
// Y plane
ByteBuffer yBuffer = planes[0].getBuffer();
yBuffer.put(data, 0, width * height);

// U plane
ByteBuffer uBuffer = planes[1].getBuffer();
int uOffset = width * height;
uBuffer.put(data, uOffset, (width * height) / 4);

// V plane
ByteBuffer vBuffer = planes[2].getBuffer();
int vOffset = uOffset + (width * height) / 4;
vBuffer.put(data, vOffset, (width * height) / 4);
return InputImage.fromMediaImage(image, rotationDegrees);
}
result.error("InputImageConverterError", "ImageFormat is not supported.", null);
// TODO: Use InputImage.fromMediaImage, which supports more types, e.g. IMAGE_FORMAT_YUV_420_888.
// See https://developers.google.com/android/reference/com/google/mlkit/vision/common/InputImage#fromMediaImage(android.media.Image,%20int)
return null;
} catch (Exception e) {
Log.e("ImageError", "Getting Image failed");
@@ -133,4 +168,11 @@ public static InputImage getInputImageFromData(Map<String, Object> imageData,
}
}

@Override
public void close() {
if (writer != null) {
writer.close();
}
}

}
19 changes: 13 additions & 6 deletions ...it_face_detection/android/src/main/java/com/google_mlkit_face_detection/FaceDetector.java
View file
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
import io.flutter.plugin.common.MethodChannel;

class FaceDetector implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startFaceDetector";
private static final String CLOSE = "vision#closeFaceDetector";

@@ -52,9 +53,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = (Map<String, Object>) call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null)
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
com.google.mlkit.vision.face.FaceDetector detector = instances.get(id);
@@ -115,7 +118,10 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
result.success(faces);
})
.addOnFailureListener(
e -> result.error("FaceDetectorError", e.toString(), null));
e -> result.error("FaceDetectorError", e.toString(), null))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());

}

private FaceDetectorOptions parseOptions(Map<String, Object> options) {
@@ -206,7 +212,7 @@ private Map<String, List<double[]>> getContourData(Face face) {
private double[] landmarkPosition(Face face, int landmarkInt) {
FaceLandmark landmark = face.getLandmark(landmarkInt);
if (landmark != null) {
return new double[] { landmark.getPosition().x, landmark.getPosition().y };
return new double[]{landmark.getPosition().x, landmark.getPosition().y};
}
return null;
}
@@ -217,7 +223,7 @@ private List<double[]> contourPosition(Face face, int contourInt) {
List<PointF> contourPoints = contour.getPoints();
List<double[]> result = new ArrayList<>();
for (int i = 0; i < contourPoints.size(); i++) {
result.add(new double[] { contourPoints.get(i).x, contourPoints.get(i).y });
result.add(new double[]{contourPoints.get(i).x, contourPoints.get(i).y});
}
return result;
}
@@ -227,8 +233,9 @@ private List<double[]> contourPosition(Face face, int contourInt) {
private void closeDetector(MethodCall call) {
String id = call.argument("id");
com.google.mlkit.vision.face.FaceDetector detector = instances.get(id);
if (detector == null)
if (detector == null) {
return;
}
detector.close();
instances.remove(id);
}
41 changes: 25 additions & 16 deletions ...etection/android/src/main/java/com/google_mlkit_face_mesh_detection/FaceMeshDetector.java
View file
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@
import io.flutter.plugin.common.MethodChannel;

class FaceMeshDetector implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startFaceMeshDetector";
private static final String CLOSE = "vision#closeFaceMeshDetector";

@@ -51,8 +52,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = (Map<String, Object>) call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null) return;
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id);
@@ -104,18 +108,18 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
meshData.put("triangles", triangles);

int[] types = {
FaceMesh.FACE_OVAL,
FaceMesh.LEFT_EYEBROW_TOP,
FaceMesh.LEFT_EYEBROW_BOTTOM,
FaceMesh.RIGHT_EYEBROW_TOP,
FaceMesh.RIGHT_EYEBROW_BOTTOM,
FaceMesh.LEFT_EYE,
FaceMesh.RIGHT_EYE,
FaceMesh.UPPER_LIP_TOP,
FaceMesh.UPPER_LIP_BOTTOM,
FaceMesh.LOWER_LIP_TOP,
FaceMesh.LOWER_LIP_BOTTOM,
FaceMesh.NOSE_BRIDGE
FaceMesh.FACE_OVAL,
FaceMesh.LEFT_EYEBROW_TOP,
FaceMesh.LEFT_EYEBROW_BOTTOM,
FaceMesh.RIGHT_EYEBROW_TOP,
FaceMesh.RIGHT_EYEBROW_BOTTOM,
FaceMesh.LEFT_EYE,
FaceMesh.RIGHT_EYE,
FaceMesh.UPPER_LIP_TOP,
FaceMesh.UPPER_LIP_BOTTOM,
FaceMesh.LOWER_LIP_TOP,
FaceMesh.LOWER_LIP_BOTTOM,
FaceMesh.NOSE_BRIDGE
};
Map<Integer, List<Map<String, Object>>> contours = new HashMap<>();
for (int type : types) {
@@ -129,7 +133,10 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)
result.success(faceMeshes);
})
.addOnFailureListener(
e -> result.error("FaceMeshDetectorError", e.toString(), null));
e -> result.error("FaceMeshDetectorError", e.toString(), null))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());

}

private List<Map<String, Object>> pointsToList(List<FaceMeshPoint> points) {
@@ -152,7 +159,9 @@ private Map<String, Object> pointToMap(FaceMeshPoint point) {
private void closeDetector(MethodCall call) {
String id = call.argument("id");
com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id);
if (detector == null) return;
if (detector == null) {
return;
}
detector.close();
instances.remove(id);
}
16 changes: 12 additions & 4 deletions ...ge_labeling/android/src/main/java/com/google_mlkit_image_labeling/ImageLabelDetector.java
View file
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@
import io.flutter.plugin.common.MethodChannel;

public class ImageLabelDetector implements MethodChannel.MethodCallHandler {

private static final String START = "vision#startImageLabelDetector";
private static final String CLOSE = "vision#closeImageLabelDetector";
private static final String MANAGE = "vision#manageFirebaseModels";
@@ -59,8 +60,11 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result

private void handleDetection(MethodCall call, final MethodChannel.Result result) {
Map<String, Object> imageData = call.argument("imageData");
InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result);
if (inputImage == null) return;
InputImageConverter converter = new InputImageConverter();
InputImage inputImage = converter.getInputImageFromData(imageData, context, result);
if (inputImage == null) {
return;
}

String id = call.argument("id");
ImageLabeler imageLabeler = instances.get(id);
@@ -106,7 +110,9 @@ private void handleDetection(MethodCall call, final MethodChannel.Result result)

result.success(labels);
})
.addOnFailureListener(e -> result.error("ImageLabelDetectorError", e.toString(), null));
.addOnFailureListener(e -> result.error("ImageLabelDetectorError", e.toString(), e))
// Closing is necessary for both success and failure.
.addOnCompleteListener(r -> converter.close());
}

//Labeler options that are provided to default image labeler(uses inbuilt model).
@@ -152,7 +158,9 @@ private CustomImageLabelerOptions getRemoteOptions(Map<String, Object> labelerOp
private void closeDetector(MethodCall call) {
String id = call.argument("id");
ImageLabeler imageLabeler = instances.get(id);
if (imageLabeler == null) return;
if (imageLabeler == null) {
return;
}
imageLabeler.close();
instances.remove(id);
}
Loading