diff --git a/mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart b/mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart index c95a5e0d5..c63ef51c2 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_detection/detection.dart @@ -1,7 +1,24 @@ -import 'dart:math' show sqrt, pow; +import 'dart:math' show max, min, pow, sqrt; import "package:photos/face/model/dimension.dart"; +enum FaceDirection { left, right, straight } + +extension FaceDirectionExtension on FaceDirection { + String toDirectionString() { + switch (this) { + case FaceDirection.left: + return 'L'; + case FaceDirection.right: + return 'R'; + case FaceDirection.straight: + return 'S'; + default: + throw Exception('Unknown FaceDirection'); + } + } +} + abstract class Detection { final double score; @@ -426,6 +443,37 @@ class FaceDetectionAbsolute extends Detection { /// The height of the bounding box of the face detection, in number of pixels, range [0, imageHeight]. double get height => yMaxBox - yMinBox; + + FaceDirection getFaceDirection() { + final double eyeDistanceX = (rightEye[0] - leftEye[0]).abs(); + final double eyeDistanceY = (rightEye[1] - leftEye[1]).abs(); + final double mouthDistanceY = (rightMouth[1] - leftMouth[1]).abs(); + + final bool faceIsUpright = + (max(leftEye[1], rightEye[1]) + 0.5 * eyeDistanceY < nose[1]) && + (nose[1] + 0.5 * mouthDistanceY < min(leftMouth[1], rightMouth[1])); + + final bool noseStickingOutLeft = (nose[0] < min(leftEye[0], rightEye[0])) && + (nose[0] < min(leftMouth[0], rightMouth[0])); + final bool noseStickingOutRight = + (nose[0] > max(leftEye[0], rightEye[0])) && + (nose[0] > max(leftMouth[0], rightMouth[0])); + + final bool noseCloseToLeftEye = + (nose[0] - leftEye[0]).abs() < 0.2 * eyeDistanceX; + final bool noseCloseToRightEye = + (nose[0] - rightEye[0]).abs() < 0.2 * eyeDistanceX; + + // if (faceIsUpright && (noseStickingOutLeft || noseCloseToLeftEye)) { + if (noseStickingOutLeft || (faceIsUpright && noseCloseToLeftEye)) { + return FaceDirection.left; + // } else if (faceIsUpright && (noseStickingOutRight || noseCloseToRightEye)) { + } else if (noseStickingOutRight || (faceIsUpright && noseCloseToRightEye)) { + return FaceDirection.right; + } + + return FaceDirection.straight; + } } List relativeToAbsoluteDetections({ diff --git a/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart index 43f6b252d..a2a28c362 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_filtering/blur_detection_service.dart @@ -1,4 +1,5 @@ import 'package:logging/logging.dart'; +import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart"; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; class BlurDetectionService { @@ -12,8 +13,10 @@ class BlurDetectionService { Future<(bool, double)> predictIsBlurGrayLaplacian( List> grayImage, { int threshold = kLaplacianThreshold, + FaceDirection faceDirection = FaceDirection.straight, }) async { - final List> laplacian = _applyLaplacian(grayImage); + final List> laplacian = + _applyLaplacian(grayImage, faceDirection: faceDirection); final double variance = _calculateVariance(laplacian); _logger.info('Variance: $variance'); return (variance < threshold, variance); @@ -46,43 +49,80 @@ class BlurDetectionService { return variance; } - List> _padImage(List> image) { + List> _padImage( + List> image, { + int removeSideColumns = 56, + FaceDirection faceDirection = FaceDirection.straight, + }) { + // Exception is removeSideColumns is not even + if (removeSideColumns % 2 != 0) { + throw Exception('removeSideColumns must be even'); + } + final int numRows = image.length; final int numCols = image[0].length; + final int paddedNumCols = numCols + 2 - removeSideColumns; + final int paddedNumRows = numRows + 2; // Create a new matrix with extra padding final List> paddedImage = List.generate( - numRows + 2, - (i) => List.generate(numCols + 2, (j) => 0, growable: false), + paddedNumRows, + (i) => List.generate( + paddedNumCols, + (j) => 0, + growable: false, + ), growable: false, ); - // Copy original image into the center of the padded image - for (int i = 0; i < numRows; i++) { - for (int j = 0; j < numCols; j++) { - paddedImage[i + 1][j + 1] = image[i][j]; + // Copy original image into the center of the padded image, taking into account the face direction + if (faceDirection == FaceDirection.straight) { + for (int i = 0; i < numRows; i++) { + for (int j = 0; j < (paddedNumCols - 2); j++) { + paddedImage[i + 1][j + 1] = + image[i][j + (removeSideColumns / 2).round()]; + } + } + // If the face is facing left, we only take the right side of the face image + } else if (faceDirection == FaceDirection.left) { + for (int i = 0; i < numRows; i++) { + for (int j = 0; j < (paddedNumCols - 2); j++) { + paddedImage[i + 1][j + 1] = image[i][j + removeSideColumns]; + } + } + // If the face is facing right, we only take the left side of the face image + } else if (faceDirection == FaceDirection.right) { + for (int i = 0; i < numRows; i++) { + for (int j = 0; j < (paddedNumCols - 2); j++) { + paddedImage[i + 1][j + 1] = image[i][j]; + } } } // Reflect padding // Top and bottom rows - for (int j = 1; j <= numCols; j++) { + for (int j = 1; j <= (paddedNumCols - 2); j++) { paddedImage[0][j] = paddedImage[2][j]; // Top row paddedImage[numRows + 1][j] = paddedImage[numRows - 1][j]; // Bottom row } // Left and right columns for (int i = 0; i < numRows + 2; i++) { paddedImage[i][0] = paddedImage[i][2]; // Left column - paddedImage[i][numCols + 1] = paddedImage[i][numCols - 1]; // Right column + paddedImage[i][paddedNumCols - 1] = + paddedImage[i][paddedNumCols - 3]; // Right column } return paddedImage; } - List> _applyLaplacian(List> image) { - final List> paddedImage = _padImage(image); - final int numRows = image.length; - final int numCols = image[0].length; + List> _applyLaplacian( + List> image, { + FaceDirection faceDirection = FaceDirection.straight, + }) { + final List> paddedImage = + _padImage(image, faceDirection: faceDirection); + final int numRows = paddedImage.length - 2; + final int numCols = paddedImage[0].length - 2; final List> outputImage = List.generate( numRows, (i) => List.generate(numCols, (j) => 0, growable: false), diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index ab29eb919..7ce10e306 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -1099,19 +1099,16 @@ Future<(Float32List, List, List, List, Size)> imageHeight: image.height, ); - final List>> faceLandmarks = - absoluteFaces.map((face) => face.allKeypoints).toList(); - final alignedImagesFloat32List = - Float32List(3 * width * height * faceLandmarks.length); + Float32List(3 * width * height * absoluteFaces.length); final alignmentResults = []; final isBlurs = []; final blurValues = []; int alignedImageIndex = 0; - for (final faceLandmark in faceLandmarks) { + for (final face in absoluteFaces) { final (alignmentResult, correctlyEstimated) = - SimilarityTransform.instance.estimate(faceLandmark); + SimilarityTransform.instance.estimate(face.allKeypoints); if (!correctlyEstimated) { alignedImageIndex += 3 * width * height; alignmentResults.add(AlignmentResult.empty()); @@ -1137,7 +1134,7 @@ Future<(Float32List, List, List, List, Size)> final grayscalems = blurDetectionStopwatch.elapsedMilliseconds; log('creating grayscale matrix took $grayscalems ms'); final (isBlur, blurValue) = await BlurDetectionService.instance - .predictIsBlurGrayLaplacian(faceGrayMatrix); + .predictIsBlurGrayLaplacian(faceGrayMatrix, faceDirection: face.getFaceDirection()); final blurms = blurDetectionStopwatch.elapsedMilliseconds - grayscalems; log('blur detection took $blurms ms'); log(