diff --git a/packages/firebase_ml_vision/CHANGELOG.md b/packages/firebase_ml_vision/CHANGELOG.md index f898f817ccd6..6632c33013d0 100644 --- a/packages/firebase_ml_vision/CHANGELOG.md +++ b/packages/firebase_ml_vision/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.1 + +* Add capability to create image from bytes. + ## 0.2.0+2 * Fix bug with empty text object. diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index 45cb19fea3c2..3ca61ee4890a 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -2,6 +2,7 @@ import android.net.Uri; import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugin.common.MethodChannel.MethodCallHandler; @@ -29,55 +30,74 @@ public static void registerWith(Registrar registrar) { @Override public void onMethodCall(MethodCall call, Result result) { Map options = call.argument("options"); + FirebaseVisionImage image; + Map imageData = call.arguments(); + try { + image = dataToVisionImage(imageData); + } catch (IOException exception) { + result.error("MLVisionDetectorIOError", exception.getLocalizedMessage(), null); + return; + } + switch (call.method) { case "BarcodeDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - BarcodeDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("barcodeDetectorIOError", e.getLocalizedMessage(), null); - } + BarcodeDetector.instance.handleDetection(image, options, result); break; case "FaceDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - FaceDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("faceDetectorIOError", e.getLocalizedMessage(), null); - } + FaceDetector.instance.handleDetection(image, options, result); break; case "LabelDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - LabelDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("labelDetectorIOError", e.getLocalizedMessage(), null); - } + LabelDetector.instance.handleDetection(image, options, result); break; case "CloudLabelDetector#detectInImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - CloudLabelDetector.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("cloudLabelDetectorIOError", e.getLocalizedMessage(), null); - } + CloudLabelDetector.instance.handleDetection(image, options, result); break; case "TextRecognizer#processImage": - try { - image = filePathToVisionImage((String) call.argument("path")); - TextRecognizer.instance.handleDetection(image, options, result); - } catch (IOException e) { - result.error("textRecognizerIOError", e.getLocalizedMessage(), null); - } + TextRecognizer.instance.handleDetection(image, options, result); break; default: result.notImplemented(); } } - private FirebaseVisionImage filePathToVisionImage(String path) throws IOException { - File file = new File(path); - return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file)); + private FirebaseVisionImage dataToVisionImage(Map imageData) throws IOException { + String imageType = (String) imageData.get("type"); + + switch (imageType) { + case "file": + File file = new File((String) imageData.get("path")); + return FirebaseVisionImage.fromFilePath(registrar.context(), Uri.fromFile(file)); + case "bytes": + @SuppressWarnings("unchecked") + Map metadataData = (Map) imageData.get("metadata"); + + FirebaseVisionImageMetadata metadata = + new FirebaseVisionImageMetadata.Builder() + .setWidth((int) (double) metadataData.get("width")) + .setHeight((int) (double) metadataData.get("height")) + .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_NV21) + .setRotation(getRotation((int) metadataData.get("rotation"))) + .build(); + + return FirebaseVisionImage.fromByteArray((byte[]) imageData.get("bytes"), metadata); + default: + throw new IllegalArgumentException(String.format("No image type for: %s", imageType)); + } + } + + private int getRotation(int rotation) { + switch (rotation) { + case 0: + return FirebaseVisionImageMetadata.ROTATION_0; + case 90: + return FirebaseVisionImageMetadata.ROTATION_90; + case 180: + return FirebaseVisionImageMetadata.ROTATION_180; + case 270: + return FirebaseVisionImageMetadata.ROTATION_270; + default: + throw new IllegalArgumentException(String.format("No rotation for: %d", rotation)); + } } } diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index 2b51f3d6a0ff..567d2b0b1d70 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -36,7 +36,7 @@ - (instancetype)init { } - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result { - FIRVisionImage *image = [self filePathToVisionImage:call.arguments[@"path"]]; + FIRVisionImage *image = [self dataToVisionImage:call.arguments]; NSDictionary *options = call.arguments[@"options"]; if ([@"BarcodeDetector#detectInImage" isEqualToString:call.method]) { [BarcodeDetector handleDetection:image options:options result:result]; @@ -53,8 +53,74 @@ - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result } } -- (FIRVisionImage *)filePathToVisionImage:(NSString *)path { - UIImage *image = [UIImage imageWithContentsOfFile:path]; - return [[FIRVisionImage alloc] initWithImage:image]; +- (FIRVisionImage *)dataToVisionImage:(NSDictionary *)imageData { + NSString *imageType = imageData[@"type"]; + + if ([@"file" isEqualToString:imageType]) { + UIImage *image = [UIImage imageWithContentsOfFile:imageData[@"path"]]; + return [[FIRVisionImage alloc] initWithImage:image]; + } else if ([@"bytes" isEqualToString:imageType]) { + FlutterStandardTypedData *byteData = imageData[@"bytes"]; + NSData *imageBytes = byteData.data; + + NSDictionary *metadata = imageData[@"metadata"]; + NSArray *planeData = metadata[@"planeData"]; + size_t planeCount = planeData.count; + + size_t widths[planeCount]; + size_t heights[planeCount]; + size_t bytesPerRows[planeCount]; + + void *baseAddresses[planeCount]; + baseAddresses[0] = (void *)imageBytes.bytes; + + size_t lastAddressIndex = 0; // Used to get base address for each plane + for (int i = 0; i < planeCount; i++) { + NSDictionary *plane = planeData[i]; + + NSNumber *width = plane[@"width"]; + NSNumber *height = plane[@"height"]; + NSNumber *bytesPerRow = plane[@"bytesPerRow"]; + + widths[i] = width.unsignedLongValue; + heights[i] = height.unsignedLongValue; + bytesPerRows[i] = bytesPerRow.unsignedLongValue; + + if (i > 0) { + size_t addressIndex = lastAddressIndex + heights[i - 1] * bytesPerRows[i - 1]; + baseAddresses[i] = (void *)imageBytes.bytes + addressIndex; + lastAddressIndex = addressIndex; + } + } + + NSNumber *width = metadata[@"width"]; + NSNumber *height = metadata[@"height"]; + + NSNumber *rawFormat = metadata[@"rawFormat"]; + FourCharCode format = FOUR_CHAR_CODE(rawFormat.unsignedIntValue); + + CVPixelBufferRef pxbuffer = NULL; + CVPixelBufferCreateWithPlanarBytes(kCFAllocatorDefault, width.unsignedLongValue, + height.unsignedLongValue, format, NULL, imageBytes.length, 2, + baseAddresses, widths, heights, bytesPerRows, NULL, NULL, + NULL, &pxbuffer); + + CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pxbuffer]; + + CIContext *temporaryContext = [CIContext contextWithOptions:nil]; + CGImageRef videoImage = + [temporaryContext createCGImage:ciImage + fromRect:CGRectMake(0, 0, CVPixelBufferGetWidth(pxbuffer), + CVPixelBufferGetHeight(pxbuffer))]; + + UIImage *uiImage = [UIImage imageWithCGImage:videoImage]; + CGImageRelease(videoImage); + return [[FIRVisionImage alloc] initWithImage:uiImage]; + } else { + NSString *errorReason = [NSString stringWithFormat:@"No image type for: %@", imageType]; + @throw [NSException exceptionWithName:NSInvalidArgumentException + reason:errorReason + userInfo:nil]; + } } @end diff --git a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart index 8e228bb65229..00f85da5bfe7 100644 --- a/packages/firebase_ml_vision/lib/firebase_ml_vision.dart +++ b/packages/firebase_ml_vision/lib/firebase_ml_vision.dart @@ -7,6 +7,8 @@ library firebase_ml_vision; import 'dart:async'; import 'dart:io'; import 'dart:math'; +import 'dart:typed_data'; +import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; diff --git a/packages/firebase_ml_vision/lib/src/barcode_detector.dart b/packages/firebase_ml_vision/lib/src/barcode_detector.dart index 8ceedbbe475a..d8705aaeaad6 100644 --- a/packages/firebase_ml_vision/lib/src/barcode_detector.dart +++ b/packages/firebase_ml_vision/lib/src/barcode_detector.dart @@ -189,11 +189,10 @@ class BarcodeDetector extends FirebaseVisionDetector { final List reply = await FirebaseVision.channel.invokeMethod( 'BarcodeDetector#detectInImage', { - 'path': visionImage.imageFile.path, 'options': { 'barcodeFormats': options.barcodeFormats.value, }, - }, + }..addAll(visionImage._serialize()), ); final List barcodes = []; diff --git a/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart b/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart index 4a4144ecd014..58c7c8885fe5 100644 --- a/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart +++ b/packages/firebase_ml_vision/lib/src/cloud_detector_options.dart @@ -29,7 +29,7 @@ class CloudDetectorOptions { /// The type of model to use for the detection. final CloudModelType modelType; - Map _toMap() => { + Map _serialize() => { 'maxResults': maxResults, 'modelType': _enumToString(modelType), }; diff --git a/packages/firebase_ml_vision/lib/src/face_detector.dart b/packages/firebase_ml_vision/lib/src/face_detector.dart index 560f2a7448cf..7d87377d840f 100644 --- a/packages/firebase_ml_vision/lib/src/face_detector.dart +++ b/packages/firebase_ml_vision/lib/src/face_detector.dart @@ -44,7 +44,6 @@ class FaceDetector extends FirebaseVisionDetector { final List reply = await FirebaseVision.channel.invokeMethod( 'FaceDetector#detectInImage', { - 'path': visionImage.imageFile.path, 'options': { 'enableClassification': options.enableClassification, 'enableLandmarks': options.enableLandmarks, @@ -52,7 +51,7 @@ class FaceDetector extends FirebaseVisionDetector { 'minFaceSize': options.minFaceSize, 'mode': _enumToString(options.mode), }, - }, + }..addAll(visionImage._serialize()), ); final List faces = []; diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 31bd400494b4..9525db5f4a3a 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -4,6 +4,13 @@ part of firebase_ml_vision; +enum _ImageType { file, bytes } + +/// Indicates the image rotation. +/// +/// Rotation is counter-clockwise. +enum ImageRotation { rotation0, rotation90, rotation180, rotation270 } + /// The Firebase machine learning vision API. /// /// You can get an instance by calling [FirebaseVision.instance] and then get @@ -56,22 +63,170 @@ class FirebaseVision { /// /// Create an instance by calling one of the factory constructors. class FirebaseVisionImage { - FirebaseVisionImage._(this.imageFile); + FirebaseVisionImage._({ + @required _ImageType type, + FirebaseVisionImageMetadata metadata, + File imageFile, + Uint8List bytes, + }) : _imageFile = imageFile, + _metadata = metadata, + _bytes = bytes, + _type = type; /// Construct a [FirebaseVisionImage] from a file. factory FirebaseVisionImage.fromFile(File imageFile) { assert(imageFile != null); - return FirebaseVisionImage._(imageFile); + return FirebaseVisionImage._( + type: _ImageType.file, + imageFile: imageFile, + ); } /// Construct a [FirebaseVisionImage] from a file path. factory FirebaseVisionImage.fromFilePath(String imagePath) { assert(imagePath != null); - return FirebaseVisionImage._(File(imagePath)); + return FirebaseVisionImage._( + type: _ImageType.file, + imageFile: File(imagePath), + ); + } + + /// Construct a [FirebaseVisionImage] from a list of bytes. + /// + /// Expects `android.graphics.ImageFormat.NV21` format on Android. Note: + /// Concatenating the planes of `android.graphics.ImageFormat.YUV_420_888` + /// into a single plane, converts it to `android.graphics.ImageFormat.NV21`. + /// + /// Expects `kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange` or any other + /// planar format on iOS. + factory FirebaseVisionImage.fromBytes( + Uint8List bytes, + FirebaseVisionImageMetadata metadata, + ) { + assert(bytes != null); + assert(metadata != null); + return FirebaseVisionImage._( + type: _ImageType.bytes, + bytes: bytes, + metadata: metadata, + ); + } + + final Uint8List _bytes; + final File _imageFile; + final FirebaseVisionImageMetadata _metadata; + final _ImageType _type; + + Map _serialize() => { + 'type': _enumToString(_type), + 'bytes': _bytes, + 'path': _imageFile?.path, + 'metadata': _type == _ImageType.bytes ? _metadata._serialize() : null, + }; +} + +/// Plane attributes to create the image buffer on iOS. +/// +/// When using iOS, [bytesPerRow], [height], and [width] throw [AssertionError] +/// if `null`. +class FirebaseVisionImagePlaneMetadata { + FirebaseVisionImagePlaneMetadata({ + @required this.bytesPerRow, + @required this.height, + @required this.width, + }) : assert(defaultTargetPlatform == TargetPlatform.iOS + ? bytesPerRow != null + : true), + assert(defaultTargetPlatform == TargetPlatform.iOS + ? height != null + : true), + assert( + defaultTargetPlatform == TargetPlatform.iOS ? width != null : true); + + /// The row stride for this color plane, in bytes. + final int bytesPerRow; + + /// Height of the pixel buffer on iOS. + final int height; + + /// Width of the pixel buffer on iOS. + final int width; + + Map _serialize() => { + 'bytesPerRow': bytesPerRow, + 'height': height, + 'width': width, + }; +} + +/// Image metadata used by [FirebaseVision] detectors. +/// +/// [rotation] defaults to [ImageRotation.rotation0]. Currently only rotates on +/// Android. +/// +/// When using iOS, [rawFormat] and [planeData] throw [AssertionError] if +/// `null`. +class FirebaseVisionImageMetadata { + FirebaseVisionImageMetadata({ + @required this.size, + @required this.rawFormat, + @required this.planeData, + this.rotation = ImageRotation.rotation0, + }) : assert(size != null), + assert(defaultTargetPlatform == TargetPlatform.iOS + ? rawFormat != null + : true), + assert(defaultTargetPlatform == TargetPlatform.iOS + ? planeData != null + : true); + + /// Size of the image in pixels. + final Size size; + + /// Rotation of the image for Android. + /// + /// Not currently used on iOS. + final ImageRotation rotation; + + /// Raw version of the format from the iOS platform. + /// + /// Since iOS can use any planar format, this format will be used to create + /// the image buffer on iOS. + /// + /// On iOS, this is a `FourCharCode` constant from Pixel Format Identifiers. + /// See https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers?language=objc + /// + /// Not used on Android. + final dynamic rawFormat; + + /// The plane attributes to create the image buffer on iOS. + /// + /// Not used on Android. + final List planeData; + + int _imageRotationToInt(ImageRotation rotation) { + switch (rotation) { + case ImageRotation.rotation90: + return 90; + case ImageRotation.rotation180: + return 180; + case ImageRotation.rotation270: + return 270; + default: + assert(rotation == ImageRotation.rotation0); + return 0; + } } - /// The file location of the image. - final File imageFile; + Map _serialize() => { + 'width': size.width, + 'height': size.height, + 'rotation': _imageRotationToInt(rotation), + 'rawFormat': rawFormat, + 'planeData': planeData + .map((FirebaseVisionImagePlaneMetadata plane) => plane._serialize()) + .toList(), + }; } /// Abstract class for detectors in [FirebaseVision] API. diff --git a/packages/firebase_ml_vision/lib/src/label_detector.dart b/packages/firebase_ml_vision/lib/src/label_detector.dart index 8511724406d0..9adaaae80b43 100644 --- a/packages/firebase_ml_vision/lib/src/label_detector.dart +++ b/packages/firebase_ml_vision/lib/src/label_detector.dart @@ -34,11 +34,10 @@ class LabelDetector extends FirebaseVisionDetector { final List reply = await FirebaseVision.channel.invokeMethod( 'LabelDetector#detectInImage', { - 'path': visionImage.imageFile.path, 'options': { 'confidenceThreshold': options.confidenceThreshold, }, - }, + }..addAll(visionImage._serialize()), ); final List