Face thumbnail generation without canvas

mobile/lib/utils/face/face_box_crop.dart

@@ -16,15 +16,19 @@ Future<Map<String, Uint8List>?> getFaceCrops(
   EnteFile file,
   Map<String, FaceBox> faceBoxeMap,
 ) async {
-  late Uint8List? ioFileBytes;
+  late String? imagePath;
   if (file.fileType != FileType.video) {
     final File? ioFile = await getFile(file);
     if (ioFile == null) {
       return null;
     }
-    ioFileBytes = await ioFile.readAsBytes();
+    imagePath = ioFile.path;
   } else {
-    ioFileBytes = await getThumbnail(file);
+    final thumbnail = await getThumbnailForUploadedFile(file);
+    if (thumbnail == null) {
+      return null;
+    }
+    imagePath = thumbnail.path;
   }
   final List<String> faceIds = [];
   final List<FaceBox> faceBoxes = [];
@@ -34,7 +38,7 @@ Future<Map<String, Uint8List>?> getFaceCrops(
   }
   final List<Uint8List> faceCrop =
       await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
-    ioFileBytes!,
+    imagePath,
     faceBoxes,
   );
   final Map<String, Uint8List> result = {};

mobile/lib/utils/image_ml_isolate.dart

@@ -1,8 +1,10 @@
 import 'dart:async';
+import "dart:io" show File;
 import 'dart:isolate';
 import 'dart:typed_data' show Float32List, Uint8List;
 import 'dart:ui';
 
+import "package:flutter/rendering.dart";
 import 'package:flutter_isolate/flutter_isolate.dart';
 import "package:logging/logging.dart";
 import "package:photos/face/model/box.dart";
@@ -13,13 +15,13 @@ import "package:photos/utils/image_ml_util.dart";
 import "package:synchronized/synchronized.dart";
 
 enum ImageOperation {
+  @Deprecated("No longer using BlazeFace`")
   preprocessBlazeFace,
   preprocessYoloOnnx,
   preprocessFaceAlign,
   preprocessMobileFaceNet,
   preprocessMobileFaceNetOnnx,
-  generateFaceThumbnail,
-  generateFaceThumbnailsForImage,
+  generateFaceThumbnails,
   cropAndPadFace,
 }
 
@@ -205,23 +207,14 @@ class ImageMlIsolate {
               'originalWidth': originalSize.width,
               'originalHeight': originalSize.height,
             });
-          case ImageOperation.generateFaceThumbnail:
-            final imageData = args['imageData'] as Uint8List;
-            final faceDetectionJson =
-                args['faceDetection'] as Map<String, dynamic>;
-            final faceDetection =
-                FaceDetectionRelative.fromJson(faceDetectionJson);
-            final Uint8List result =
-                await generateFaceThumbnailFromData(imageData, faceDetection);
-            sendPort.send(<dynamic>[result]);
-          case ImageOperation.generateFaceThumbnailsForImage:
-            final imageData = args['imageData'] as Uint8List;
+          case ImageOperation.generateFaceThumbnails:
+            final imagePath = args['imagePath'] as String;
+            final Uint8List imageData = await File(imagePath).readAsBytes();
             final faceBoxesJson =
                 args['faceBoxesList'] as List<Map<String, dynamic>>;
             final List<FaceBox> faceBoxes =
                 faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList();
-            final List<Uint8List> results =
-                await generateFaceThumbnailsFromDataAndDetections(
+            final List<Uint8List> results = await generateFaceThumbnails(
               imageData,
               faceBoxes,
             );
@@ -471,44 +464,28 @@ class ImageMlIsolate {
     return (inputs, alignmentResults, isBlurs, blurValues, originalSize);
   }
 
-  /// Generates a face thumbnail from [imageData] and a [faceDetection].
-  ///
-  /// Uses [generateFaceThumbnailFromData] inside the isolate.
-  Future<Uint8List> generateFaceThumbnail(
-    Uint8List imageData,
-    FaceDetectionRelative faceDetection,
-  ) async {
-    return await _runInIsolate(
-      (
-        ImageOperation.generateFaceThumbnail,
-        {
-          'imageData': imageData,
-          'faceDetection': faceDetection.toJson(),
-        },
-      ),
-    ).then((value) => value[0] as Uint8List);
-  }
-
   /// Generates face thumbnails for all [faceBoxes] in [imageData].
   ///
-  /// Uses [generateFaceThumbnailsFromDataAndDetections] inside the isolate.
+  /// Uses [generateFaceThumbnails] inside the isolate.
   Future<List<Uint8List>> generateFaceThumbnailsForImage(
-    Uint8List imageData,
+    String imagePath,
     List<FaceBox> faceBoxes,
   ) async {
     final List<Map<String, dynamic>> faceBoxesJson =
         faceBoxes.map((box) => box.toJson()).toList();
     return await _runInIsolate(
       (
-        ImageOperation.generateFaceThumbnailsForImage,
+        ImageOperation.generateFaceThumbnails,
         {
-          'imageData': imageData,
+          'imagePath': imagePath,
           'faceBoxesList': faceBoxesJson,
         },
       ),
     ).then((value) => value.cast<Uint8List>());
   }
 
+  @Deprecated('For second pass of BlazeFace, no longer used')
+
   /// Generates cropped and padded image data from [imageData] and a [faceBox].
   ///
   /// The steps are:

mobile/lib/utils/image_ml_util.dart

@@ -15,6 +15,7 @@ import "dart:ui";
 //         ImageConfiguration;
 // import 'package:flutter/material.dart' as material show Image;
 import 'package:flutter/painting.dart' as paint show decodeImageFromList;
+import 'package:image/image.dart' as img_lib;
 import 'package:ml_linalg/linalg.dart';
 import "package:photos/face/model/box.dart";
 import 'package:photos/models/ml/ml_typedefs.dart';
@@ -36,7 +37,7 @@ Color readPixelColor(
   if (x < 0 || x >= image.width || y < 0 || y >= image.height) {
     // throw ArgumentError('Invalid pixel coordinates.');
     log('[WARNING] `readPixelColor`: Invalid pixel coordinates, out of bounds');
-    return const Color(0x00000000);
+    return const Color.fromARGB(255, 208, 16, 208);
   }
   assert(byteData.lengthInBytes == 4 * image.width * image.height);
 
@@ -44,12 +45,39 @@ Color readPixelColor(
   return Color(_rgbaToArgb(byteData.getUint32(byteOffset)));
 }
 
+void setPixelColor(
+  Size imageSize,
+  ByteData byteData,
+  int x,
+  int y,
+  Color color,
+) {
+  if (x < 0 || x >= imageSize.width || y < 0 || y >= imageSize.height) {
+    log('[WARNING] `setPixelColor`: Invalid pixel coordinates, out of bounds');
+    return;
+  }
+  assert(byteData.lengthInBytes == 4 * imageSize.width * imageSize.height);
+
+  final int byteOffset = 4 * (imageSize.width.toInt() * y + x);
+  byteData.setUint32(byteOffset, _argbToRgba(color.value));
+}
+
 int _rgbaToArgb(int rgbaColor) {
   final int a = rgbaColor & 0xFF;
   final int rgb = rgbaColor >> 8;
   return rgb + (a << 24);
 }
 
+int _argbToRgba(int argbColor) {
+  final int r = (argbColor >> 16) & 0xFF;
+  final int g = (argbColor >> 8) & 0xFF;
+  final int b = argbColor & 0xFF;
+  final int a = (argbColor >> 24) & 0xFF;
+  return (r << 24) + (g << 16) + (b << 8) + a;
+}
+
+@Deprecated('Used in TensorFlow Lite only, no longer needed')
+
 /// Creates an empty matrix with the specified shape.
 ///
 /// The `shape` argument must be a list of length 2 or 3, where the first
@@ -464,11 +492,53 @@ Future<Image> resizeAndCenterCropImage(
   return resizedImage;
 }
 
+/// Crops an [image] based on the specified [x], [y], [width] and [height].
+Future<Uint8List> cropImage(
+  Image image,
+  ByteData imgByteData, {
+  required int x,
+  required int y,
+  required int width,
+  required int height,
+}) async {
+  // final newByteData = ByteData(width * height * 4);
+  // for (var h = y; h < y + height; h++) {
+  //   for (var w = x; w < x + width; w++) {
+  //     final pixel = readPixelColor(image, imgByteData, w, y);
+  //     setPixelColor(
+  //       Size(width.toDouble(), height.toDouble()),
+  //       newByteData,
+  //       w,
+  //       h,
+  //       pixel,
+  //     );
+  //   }
+  // }
+  // final newImage =
+  //     decodeImageFromRgbaBytes(newByteData.buffer.asUint8List(), width, height);
+
+  final newImage = img_lib.Image(width: width, height: height);
+
+  for (var h = y; h < y + height; h++) {
+    for (var w = x; w < x + width; w++) {
+      final pixel = readPixelColor(image, imgByteData, w, h);
+      newImage.setPixel(
+        w - x,
+        h - y,
+        img_lib.ColorRgb8(pixel.red, pixel.green, pixel.blue),
+      );
+    }
+  }
+  final newImageDataPng = img_lib.encodePng(newImage);
+
+  return newImageDataPng;
+}
+
 /// Crops an [image] based on the specified [x], [y], [width] and [height].
 /// Optionally, the cropped image can be resized to comply with a [maxSize] and/or [minSize].
 /// Optionally, the cropped image can be rotated from the center by [rotation] radians.
 /// Optionally, the [quality] of the resizing interpolation can be specified.
-Future<Image> cropImage(
+Future<Image> cropImageWithCanvas(
   Image image, {
   required double x,
   required double y,
@@ -798,7 +868,7 @@ Future<List<Uint8List>> preprocessFaceAlignToUint8List(
       continue;
     }
     final alignmentBox = getAlignedFaceBox(alignmentResult);
-    final Image alignedFace = await cropImage(
+    final Image alignedFace = await cropImageWithCanvas(
       image,
       x: alignmentBox[0],
       y: alignmentBox[1],
@@ -886,7 +956,7 @@ Future<
       continue;
     }
     final alignmentBox = getAlignedFaceBox(alignmentResult);
-    final Image alignedFace = await cropImage(
+    final Image alignedFace = await cropImageWithCanvas(
       image,
       x: alignmentBox[0],
       y: alignmentBox[1],
@@ -970,7 +1040,7 @@ Future<(Float32List, List<AlignmentResult>, List<bool>, List<double>, Size)>
       continue;
     }
     final alignmentBox = getAlignedFaceBox(alignmentResult);
-    final Image alignedFace = await cropImage(
+    final Image alignedFace = await cropImageWithCanvas(
       image,
       x: alignmentBox[0],
       y: alignmentBox[1],
@@ -1168,29 +1238,52 @@ void warpAffineFloat32List(
   }
 }
 
-/// Generates a face thumbnail from [imageData] and a [faceDetection].
-///
-/// Returns a [Uint8List] image, in png format.
-Future<Uint8List> generateFaceThumbnailFromData(
+Future<List<Uint8List>> generateFaceThumbnails(
   Uint8List imageData,
-  FaceDetectionRelative faceDetection,
+  List<FaceBox> faceBoxes,
 ) async {
+  final stopwatch = Stopwatch()..start();
+
   final Image image = await decodeImageFromData(imageData);
+  final ByteData imgByteData = await getByteDataFromImage(image);
 
-  final Image faceThumbnail = await cropImage(
-    image,
-    x: (faceDetection.xMinBox * image.width).round() - 20,
-    y: (faceDetection.yMinBox * image.height).round() - 30,
-    width: (faceDetection.width * image.width).round() + 40,
-    height: (faceDetection.height * image.height).round() + 60,
-  );
+  // int i = 0;
+  try {
+    final List<Uint8List> faceThumbnails = [];
 
-  return await encodeImageToUint8List(
-    faceThumbnail,
-    format: ImageByteFormat.png,
-  );
+    for (final faceBox in faceBoxes) {
+      final xCrop = (faceBox.x - faceBox.width / 2).round();
+      final yCrop = (faceBox.y - faceBox.height / 2).round();
+      final widthCrop = (faceBox.width * 2).round();
+      final heightCrop = (faceBox.height * 2).round();
+      final Uint8List faceThumbnail = await cropImage(
+        image,
+        imgByteData,
+        x: xCrop,
+        y: yCrop,
+        width: widthCrop,
+        height: heightCrop,
+      );
+      // final Uint8List faceThumbnailPng = await encodeImageToUint8List(
+      //   faceThumbnail,
+      //   format: ImageByteFormat.png,
+      // );
+      faceThumbnails.add(faceThumbnail);
+      // i++;
+    }
+    stopwatch.stop();
+    log('Face thumbnail generation took: ${stopwatch.elapsedMilliseconds} ms');
+
+    return faceThumbnails;
+  } catch (e, s) {
+    log('[ImageMlUtils] Error generating face thumbnails: $e, \n stackTrace: $s');
+    // log('[ImageMlUtils] cropImage problematic input argument: ${faceBoxes[i]}');
+    rethrow;
+  }
 }
 
+@Deprecated("Old method using canvas, replaced by `generateFaceThumbnails`")
+
 /// Generates a face thumbnail from [imageData] and a [faceDetection].
 ///
 /// Returns a [Uint8List] image, in png format.
@@ -1205,7 +1298,7 @@ Future<List<Uint8List>> generateFaceThumbnailsFromDataAndDetections(
     final List<Uint8List> faceThumbnails = [];
 
     for (final faceBox in faceBoxes) {
-      final Image faceThumbnail = await cropImage(
+      final Image faceThumbnail = await cropImageWithCanvas(
         image,
         x: faceBox.x - faceBox.width / 2,
         y: faceBox.y - faceBox.height / 2,
@@ -1227,6 +1320,8 @@ Future<List<Uint8List>> generateFaceThumbnailsFromDataAndDetections(
   }
 }
 
+@Deprecated('For second pass of BlazeFace, no longer used')
+
 /// Generates cropped and padded image data from [imageData] and a [faceBox].
 ///
 /// The steps are:
@@ -1241,7 +1336,7 @@ Future<Uint8List> cropAndPadFaceData(
 ) async {
   final Image image = await decodeImageFromData(imageData);
 
-  final Image faceCrop = await cropImage(
+  final Image faceCrop = await cropImageWithCanvas(
     image,
     x: (faceBox[0] * image.width),
     y: (faceBox[1] * image.height),
@@ -1390,6 +1485,7 @@ Color getPixelBicubic(num fx, num fy, Image image, ByteData byteDataRgba) {
   return Color.fromRGBO(c0, c1, c2, 1.0);
 }
 
+@Deprecated('Old method only used in other deprecated methods')
 List<double> getAlignedFaceBox(AlignmentResult alignment) {
   final List<double> box = [
     // [xMinBox, yMinBox, xMaxBox, yMaxBox]