diff --git a/mobile/lib/face/db_model_mappers.dart b/mobile/lib/face/db_model_mappers.dart index be9596ffd..e9a3d09ed 100644 --- a/mobile/lib/face/db_model_mappers.dart +++ b/mobile/lib/face/db_model_mappers.dart @@ -4,7 +4,6 @@ import 'package:photos/face/db_fields.dart'; import "package:photos/face/model/detection.dart"; import "package:photos/face/model/face.dart"; import "package:photos/face/model/person.dart"; -import 'package:photos/face/model/person_face.dart'; import "package:photos/generated/protos/ente/common/vector.pb.dart"; int boolToSQLInt(bool? value, {bool defaultValue = false}) { diff --git a/mobile/lib/face/model/box.dart b/mobile/lib/face/model/box.dart index 1ef89144c..73d7dea38 100644 --- a/mobile/lib/face/model/box.dart +++ b/mobile/lib/face/model/box.dart @@ -1,29 +1,30 @@ -/// Bounding box of a face. -/// -/// [`x`] and [y] are the coordinates of the top left corner of the box, so the minimim values +/// Bounding box of a face. +/// +/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and /// [width] and [height] are the width and height of the box. -/// All values are in absolute pixels relative to the original image size. +/// +/// WARNING: All values are relative to the original image size, so in the range [0, 1]. class FaceBox { - final double x; - final double y; + final double xMin; + final double yMin; final double width; final double height; FaceBox({ - required this.x, - required this.y, + required this.xMin, + required this.yMin, required this.width, required this.height, }); factory FaceBox.fromJson(Map json) { return FaceBox( - x: (json['x'] is int - ? (json['x'] as int).toDouble() - : json['x'] as double), - y: (json['y'] is int - ? (json['y'] as int).toDouble() - : json['y'] as double), + xMin: (json['xMin'] is int + ? (json['xMin'] as int).toDouble() + : json['xMin'] as double), + yMin: (json['yMin'] is int + ? (json['yMin'] as int).toDouble() + : json['yMin'] as double), width: (json['width'] is int ? (json['width'] as int).toDouble() : json['width'] as double), @@ -34,8 +35,8 @@ class FaceBox { } Map toJson() => { - 'x': x, - 'y': y, + 'xMin': xMin, + 'yMin': yMin, 'width': width, 'height': height, }; diff --git a/mobile/lib/face/model/detection.dart b/mobile/lib/face/model/detection.dart index 7d5b02cc6..44acd85bb 100644 --- a/mobile/lib/face/model/detection.dart +++ b/mobile/lib/face/model/detection.dart @@ -1,6 +1,9 @@ import "package:photos/face/model/box.dart"; import "package:photos/face/model/landmark.dart"; +/// Stores the face detection data, notably the bounding box and landmarks. +/// +/// WARNING: All coordinates are relative to the image size, so in the range [0, 1]! class Detection { FaceBox box; List landmarks; @@ -10,11 +13,13 @@ class Detection { required this.landmarks, }); + bool get isEmpty => box.width == 0 && box.height == 0 && landmarks.isEmpty; + // emoty box Detection.empty() : box = FaceBox( - x: 0, - y: 0, + xMin: 0, + yMin: 0, width: 0, height: 0, ), diff --git a/mobile/lib/face/model/landmark.dart b/mobile/lib/face/model/landmark.dart index 03a68fd11..13808c56b 100644 --- a/mobile/lib/face/model/landmark.dart +++ b/mobile/lib/face/model/landmark.dart @@ -1,4 +1,6 @@ -// Class for the 'landmark' sub-object +/// Landmark coordinate data. +/// +/// WARNING: All coordinates are relative to the image size, so in the range [0, 1]! class Landmark { double x; double y; diff --git a/mobile/lib/services/face_ml/face_ml_service.dart b/mobile/lib/services/face_ml/face_ml_service.dart index a75182b1f..a5dcc72ab 100644 --- a/mobile/lib/services/face_ml/face_ml_service.dart +++ b/mobile/lib/services/face_ml/face_ml_service.dart @@ -496,35 +496,21 @@ class FaceMlService { _logger.severe( "faceDetectionImageSize or faceDetectionImageSize is null for image with " "ID: ${enteFile.uploadedFileID}"); - } - final bool useAlign = result.faceAlignmentImageSize != null && - result.faceAlignmentImageSize!.width > 0 && - result.faceAlignmentImageSize!.height > 0 && - result.onlyThumbnailUsed == false; - if (useAlign) { _logger.info( "Using aligned image size for image with ID: ${enteFile.uploadedFileID}. This size is ${result.faceAlignmentImageSize!.width}x${result.faceAlignmentImageSize!.height} compared to size of ${enteFile.width}x${enteFile.height} in the metadata", ); } for (int i = 0; i < result.faces.length; ++i) { final FaceResult faceRes = result.faces[i]; - final FaceDetectionAbsolute absoluteDetection = - faceRes.detection.toAbsolute( - imageWidth: useAlign - ? result.faceAlignmentImageSize!.width.toInt() - : enteFile.width, - imageHeight: useAlign - ? result.faceAlignmentImageSize!.height.toInt() - : enteFile.height, - ); + final FaceDetectionRelative relativeDetection = faceRes.detection; final detection = face_detection.Detection( box: FaceBox( - x: absoluteDetection.xMinBox, - y: absoluteDetection.yMinBox, - width: absoluteDetection.width, - height: absoluteDetection.height, + xMin: relativeDetection.xMinBox, + yMin: relativeDetection.yMinBox, + width: relativeDetection.width, + height: relativeDetection.height, ), - landmarks: absoluteDetection.allKeypoints + landmarks: relativeDetection.allKeypoints .map( (keypoint) => Landmark( x: keypoint[0], diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 0b85119ca..48315b745 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -1232,23 +1232,25 @@ Future> generateFaceThumbnails( ) async { final stopwatch = Stopwatch()..start(); - final Image image = await decodeImageFromData(imageData); - final ByteData imgByteData = await getByteDataFromImage(image); + final Image img = await decodeImageFromData(imageData); + final ByteData imgByteData = await getByteDataFromImage(img); try { final List faceThumbnails = []; for (final faceBox in faceBoxes) { - final int xCrop = - (faceBox.x - faceBox.width / 2).round().clamp(0, image.width); - final int yCrop = - (faceBox.y - faceBox.height / 2).round().clamp(0, image.height); - final int widthCrop = - min((faceBox.width * 2).round(), image.width - xCrop); - final int heightCrop = - min((faceBox.height * 2).round(), image.height - yCrop); + // Note that the faceBox values are relative to the image size, so we need to convert them to absolute values first + final double xMinAbs = faceBox.xMin * img.width; + final double yMinAbs = faceBox.yMin * img.height; + final double widthAbs = faceBox.width * img.width; + final double heightAbs = faceBox.height * img.height; + + final int xCrop = (xMinAbs - widthAbs / 2).round().clamp(0, img.width); + final int yCrop = (yMinAbs - heightAbs / 2).round().clamp(0, img.height); + final int widthCrop = min((widthAbs * 2).round(), img.width - xCrop); + final int heightCrop = min((heightAbs * 2).round(), img.height - yCrop); final Image faceThumbnail = await cropImage( - image, + img, imgByteData, x: xCrop, y: yCrop, @@ -1280,19 +1282,25 @@ Future> generateFaceThumbnailsFromDataAndDetections( Uint8List imageData, List faceBoxes, ) async { - final Image image = await decodeImageFromData(imageData); + final Image img = await decodeImageFromData(imageData); int i = 0; try { final List faceThumbnails = []; for (final faceBox in faceBoxes) { + // Note that the faceBox values are relative to the image size, so we need to convert them to absolute values first + final double xMinAbs = faceBox.xMin * img.width; + final double yMinAbs = faceBox.yMin * img.height; + final double widthAbs = faceBox.width * img.width; + final double heightAbs = faceBox.height * img.height; + final Image faceThumbnail = await cropImageWithCanvas( - image, - x: faceBox.x - faceBox.width / 2, - y: faceBox.y - faceBox.height / 2, - width: faceBox.width * 2, - height: faceBox.height * 2, + img, + x: xMinAbs - widthAbs / 2, + y: yMinAbs - heightAbs / 2, + width: widthAbs * 2, + height: heightAbs * 2, ); final Uint8List faceThumbnailPng = await encodeImageToUint8List( faceThumbnail,