[mob] Better blur detection handling background noise

mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart

@@ -1,7 +1,24 @@
-import 'dart:math' show sqrt, pow;
+import 'dart:math' show max, min, pow, sqrt;
 
 import "package:photos/face/model/dimension.dart";
 
+enum FaceDirection { left, right, straight }
+
+extension FaceDirectionExtension on FaceDirection {
+  String toDirectionString() {
+    switch (this) {
+      case FaceDirection.left:
+        return 'L';
+      case FaceDirection.right:
+        return 'R';
+      case FaceDirection.straight:
+        return 'S';
+      default:
+        throw Exception('Unknown FaceDirection');
+    }
+  }
+}
+
 abstract class Detection {
   final double score;
 
@@ -426,6 +443,37 @@ class FaceDetectionAbsolute extends Detection {
 
   /// The height of the bounding box of the face detection, in number of pixels, range [0, imageHeight].
   double get height => yMaxBox - yMinBox;
+
+  FaceDirection getFaceDirection() {
+    final double eyeDistanceX = (rightEye[0] - leftEye[0]).abs();
+    final double eyeDistanceY = (rightEye[1] - leftEye[1]).abs();
+    final double mouthDistanceY = (rightMouth[1] - leftMouth[1]).abs();
+
+    final bool faceIsUpright =
+        (max(leftEye[1], rightEye[1]) + 0.5 * eyeDistanceY < nose[1]) &&
+            (nose[1] + 0.5 * mouthDistanceY < min(leftMouth[1], rightMouth[1]));
+
+    final bool noseStickingOutLeft = (nose[0] < min(leftEye[0], rightEye[0])) &&
+        (nose[0] < min(leftMouth[0], rightMouth[0]));
+    final bool noseStickingOutRight =
+        (nose[0] > max(leftEye[0], rightEye[0])) &&
+            (nose[0] > max(leftMouth[0], rightMouth[0]));
+
+    final bool noseCloseToLeftEye =
+        (nose[0] - leftEye[0]).abs() < 0.2 * eyeDistanceX;
+    final bool noseCloseToRightEye =
+        (nose[0] - rightEye[0]).abs() < 0.2 * eyeDistanceX;
+
+    // if (faceIsUpright && (noseStickingOutLeft || noseCloseToLeftEye)) {
+    if (noseStickingOutLeft || (faceIsUpright && noseCloseToLeftEye)) {
+      return FaceDirection.left;
+      // } else if (faceIsUpright && (noseStickingOutRight || noseCloseToRightEye)) {
+    } else if (noseStickingOutRight || (faceIsUpright && noseCloseToRightEye)) {
+      return FaceDirection.right;
+    }
+
+    return FaceDirection.straight;
+  }
 }
 
 List<FaceDetectionAbsolute> relativeToAbsoluteDetections({

mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart

@@ -1,4 +1,5 @@
 import 'package:logging/logging.dart';
+import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart";
 import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
 
 class BlurDetectionService {
@@ -12,8 +13,10 @@ class BlurDetectionService {
   Future<(bool, double)> predictIsBlurGrayLaplacian(
     List<List<int>> grayImage, {
     int threshold = kLaplacianThreshold,
+    FaceDirection faceDirection = FaceDirection.straight,
   }) async {
-    final List<List<int>> laplacian = _applyLaplacian(grayImage);
+    final List<List<int>> laplacian =
+        _applyLaplacian(grayImage, faceDirection: faceDirection);
     final double variance = _calculateVariance(laplacian);
     _logger.info('Variance: $variance');
     return (variance < threshold, variance);
@@ -46,43 +49,80 @@ class BlurDetectionService {
     return variance;
   }
 
-  List<List<int>> _padImage(List<List<int>> image) {
+  List<List<int>> _padImage(
+    List<List<int>> image, {
+    int removeSideColumns = 56,
+    FaceDirection faceDirection = FaceDirection.straight,
+  }) {
+    // Exception is removeSideColumns is not even
+    if (removeSideColumns % 2 != 0) {
+      throw Exception('removeSideColumns must be even');
+    }
+
     final int numRows = image.length;
     final int numCols = image[0].length;
+    final int paddedNumCols = numCols + 2 - removeSideColumns;
+    final int paddedNumRows = numRows + 2;
 
     // Create a new matrix with extra padding
     final List<List<int>> paddedImage = List.generate(
-      numRows + 2,
-      (i) => List.generate(numCols + 2, (j) => 0, growable: false),
+      paddedNumRows,
+      (i) => List.generate(
+        paddedNumCols,
+        (j) => 0,
+        growable: false,
+      ),
       growable: false,
     );
 
-    // Copy original image into the center of the padded image
-    for (int i = 0; i < numRows; i++) {
-      for (int j = 0; j < numCols; j++) {
-        paddedImage[i + 1][j + 1] = image[i][j];
+    // Copy original image into the center of the padded image, taking into account the face direction
+    if (faceDirection == FaceDirection.straight) {
+      for (int i = 0; i < numRows; i++) {
+        for (int j = 0; j < (paddedNumCols - 2); j++) {
+          paddedImage[i + 1][j + 1] =
+              image[i][j + (removeSideColumns / 2).round()];
+        }
+      }
+      // If the face is facing left, we only take the right side of the face image
+    } else if (faceDirection == FaceDirection.left) {
+      for (int i = 0; i < numRows; i++) {
+        for (int j = 0; j < (paddedNumCols - 2); j++) {
+          paddedImage[i + 1][j + 1] = image[i][j + removeSideColumns];
+        }
+      }
+      // If the face is facing right, we only take the left side of the face image
+    } else if (faceDirection == FaceDirection.right) {
+      for (int i = 0; i < numRows; i++) {
+        for (int j = 0; j < (paddedNumCols - 2); j++) {
+          paddedImage[i + 1][j + 1] = image[i][j];
+        }
       }
     }
 
     // Reflect padding
     // Top and bottom rows
-    for (int j = 1; j <= numCols; j++) {
+    for (int j = 1; j <= (paddedNumCols - 2); j++) {
       paddedImage[0][j] = paddedImage[2][j]; // Top row
       paddedImage[numRows + 1][j] = paddedImage[numRows - 1][j]; // Bottom row
     }
     // Left and right columns
     for (int i = 0; i < numRows + 2; i++) {
       paddedImage[i][0] = paddedImage[i][2]; // Left column
-      paddedImage[i][numCols + 1] = paddedImage[i][numCols - 1]; // Right column
+      paddedImage[i][paddedNumCols - 1] =
+          paddedImage[i][paddedNumCols - 3]; // Right column
     }
 
     return paddedImage;
   }
 
-  List<List<int>> _applyLaplacian(List<List<int>> image) {
-    final List<List<int>> paddedImage = _padImage(image);
-    final int numRows = image.length;
-    final int numCols = image[0].length;
+  List<List<int>> _applyLaplacian(
+    List<List<int>> image, {
+    FaceDirection faceDirection = FaceDirection.straight,
+  }) {
+    final List<List<int>> paddedImage =
+        _padImage(image, faceDirection: faceDirection);
+    final int numRows = paddedImage.length - 2;
+    final int numCols = paddedImage[0].length - 2;
     final List<List<int>> outputImage = List.generate(
       numRows,
       (i) => List.generate(numCols, (j) => 0, growable: false),

mobile/lib/utils/image_ml_util.dart

@@ -1099,19 +1099,16 @@ Future<(Float32List, List<AlignmentResult>, List<bool>, List<double>, Size)>
     imageHeight: image.height,
   );
 
-  final List<List<List<double>>> faceLandmarks =
-      absoluteFaces.map((face) => face.allKeypoints).toList();
-
   final alignedImagesFloat32List =
-      Float32List(3 * width * height * faceLandmarks.length);
+      Float32List(3 * width * height * absoluteFaces.length);
   final alignmentResults = <AlignmentResult>[];
   final isBlurs = <bool>[];
   final blurValues = <double>[];
 
   int alignedImageIndex = 0;
-  for (final faceLandmark in faceLandmarks) {
+  for (final face in absoluteFaces) {
     final (alignmentResult, correctlyEstimated) =
-        SimilarityTransform.instance.estimate(faceLandmark);
+        SimilarityTransform.instance.estimate(face.allKeypoints);
     if (!correctlyEstimated) {
       alignedImageIndex += 3 * width * height;
       alignmentResults.add(AlignmentResult.empty());
@@ -1137,7 +1134,7 @@ Future<(Float32List, List<AlignmentResult>, List<bool>, List<double>, Size)>
     final grayscalems = blurDetectionStopwatch.elapsedMilliseconds;
     log('creating grayscale matrix took $grayscalems ms');
     final (isBlur, blurValue) = await BlurDetectionService.instance
-        .predictIsBlurGrayLaplacian(faceGrayMatrix);
+        .predictIsBlurGrayLaplacian(faceGrayMatrix, faceDirection: face.getFaceDirection());
     final blurms = blurDetectionStopwatch.elapsedMilliseconds - grayscalems;
     log('blur detection took $blurms ms');
     log(