diff --git a/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts b/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts index 699d53cfe..9fc0f7ad2 100644 --- a/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts +++ b/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts @@ -1,4 +1,11 @@ import { MAX_FACE_DISTANCE_PERCENT } from "constants/mlConfig"; +import { + Matrix, + applyToPoint, + compose, + scale, + translate, +} from "transformation-matrix"; import { Dimensions } from "types/image"; import { FaceDetection, @@ -12,11 +19,6 @@ import { normalizePixelBetween0And1, } from "utils/image"; import { newBox } from "utils/machineLearning"; -import { - computeTransformToBox, - transformBox, - transformPoints, -} from "utils/machineLearning/transform"; import { Box, Point } from "../../../thirdparty/face-api/classes"; // TODO(MR): onnx-yolo @@ -385,3 +387,35 @@ function getDetectionCenter(detection: FaceDetection) { return center.div({ x: 4, y: 4 }); } + +function computeTransformToBox(inBox: Box, toBox: Box): Matrix { + return compose( + translate(toBox.x, toBox.y), + scale(toBox.width / inBox.width, toBox.height / inBox.height), + ); +} + +function transformPoint(point: Point, transform: Matrix) { + const txdPoint = applyToPoint(transform, point); + return new Point(txdPoint.x, txdPoint.y); +} + +function transformPoints(points: Point[], transform: Matrix) { + return points?.map((p) => transformPoint(p, transform)); +} + +function transformBox(box: Box, transform: Matrix) { + const topLeft = transformPoint(box.topLeft, transform); + const bottomRight = transformPoint(box.bottomRight, transform); + + return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y); +} + +function newBoxFromPoints( + left: number, + top: number, + right: number, + bottom: number, +) { + return new Box({ left, top, right, bottom }); +} diff --git a/web/apps/photos/src/utils/machineLearning/faceAlign.ts b/web/apps/photos/src/utils/machineLearning/faceAlign.ts index b5117d4b0..beb98cea9 100644 --- a/web/apps/photos/src/utils/machineLearning/faceAlign.ts +++ b/web/apps/photos/src/utils/machineLearning/faceAlign.ts @@ -1,20 +1,39 @@ import { Matrix } from "ml-matrix"; import { getSimilarityTransformation } from "similarity-transformation"; -import { Dimensions } from "types/image"; import { FaceAlignment, FaceDetection } from "types/machineLearning"; -import { cropWithRotation, transform } from "utils/image"; -import { Box, Point } from "../../../thirdparty/face-api/classes"; +import { Point } from "../../../thirdparty/face-api/classes"; -export function normalizeLandmarks( - landmarks: Array<[number, number]>, - faceSize: number, -): Array<[number, number]> { - return landmarks.map((landmark) => - landmark.map((p) => p / faceSize), - ) as Array<[number, number]>; +const ARCFACE_LANDMARKS = [ + [38.2946, 51.6963], + [73.5318, 51.5014], + [56.0252, 71.7366], + [56.1396, 92.2848], +] as Array<[number, number]>; + +const ARCFACE_LANDMARKS_FACE_SIZE = 112; + +const ARC_FACE_5_LANDMARKS = [ + [38.2946, 51.6963], + [73.5318, 51.5014], + [56.0252, 71.7366], + [41.5493, 92.3655], + [70.7299, 92.2041], +] as Array<[number, number]>; + +export function getArcfaceAlignment( + faceDetection: FaceDetection, +): FaceAlignment { + const landmarkCount = faceDetection.landmarks.length; + return getFaceAlignmentUsingSimilarityTransform( + faceDetection, + normalizeLandmarks( + landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS, + ARCFACE_LANDMARKS_FACE_SIZE, + ), + ); } -export function getFaceAlignmentUsingSimilarityTransform( +function getFaceAlignmentUsingSimilarityTransform( faceDetection: FaceDetection, alignedLandmarks: Array<[number, number]>, // alignmentMethod: Versioned @@ -58,83 +77,11 @@ export function getFaceAlignmentUsingSimilarityTransform( }; } -const ARCFACE_LANDMARKS = [ - [38.2946, 51.6963], - [73.5318, 51.5014], - [56.0252, 71.7366], - [56.1396, 92.2848], -] as Array<[number, number]>; - -const ARCFACE_LANDMARKS_FACE_SIZE = 112; - -const ARC_FACE_5_LANDMARKS = [ - [38.2946, 51.6963], - [73.5318, 51.5014], - [56.0252, 71.7366], - [41.5493, 92.3655], - [70.7299, 92.2041], -] as Array<[number, number]>; - -export function getArcfaceAlignment( - faceDetection: FaceDetection, -): FaceAlignment { - const landmarkCount = faceDetection.landmarks.length; - return getFaceAlignmentUsingSimilarityTransform( - faceDetection, - normalizeLandmarks( - landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS, - ARCFACE_LANDMARKS_FACE_SIZE, - ), - ); -} - -export function getAlignedFaceBox(alignment: FaceAlignment) { - return new Box({ - x: alignment.center.x - alignment.size / 2, - y: alignment.center.y - alignment.size / 2, - width: alignment.size, - height: alignment.size, - }).round(); -} - -export function ibExtractFaceImage( - image: ImageBitmap, - alignment: FaceAlignment, +function normalizeLandmarks( + landmarks: Array<[number, number]>, faceSize: number, -): ImageBitmap { - const box = getAlignedFaceBox(alignment); - const faceSizeDimentions: Dimensions = { - width: faceSize, - height: faceSize, - }; - return cropWithRotation( - image, - box, - alignment.rotation, - faceSizeDimentions, - faceSizeDimentions, - ); -} - -// Used in MLDebugViewOnly -export function ibExtractFaceImageUsingTransform( - image: ImageBitmap, - alignment: FaceAlignment, - faceSize: number, -): ImageBitmap { - const scaledMatrix = new Matrix(alignment.affineMatrix) - .mul(faceSize) - .to2DArray(); - // log.info("scaledMatrix: ", scaledMatrix); - return transform(image, scaledMatrix, faceSize, faceSize); -} - -export function ibExtractFaceImages( - image: ImageBitmap, - alignments: Array, - faceSize: number, -): Array { - return alignments.map((alignment) => - ibExtractFaceImage(image, alignment, faceSize), - ); +): Array<[number, number]> { + return landmarks.map((landmark) => + landmark.map((p) => p / faceSize), + ) as Array<[number, number]>; } diff --git a/web/apps/photos/src/utils/machineLearning/faceCrop.ts b/web/apps/photos/src/utils/machineLearning/faceCrop.ts index ea6db2608..5486c6448 100644 --- a/web/apps/photos/src/utils/machineLearning/faceCrop.ts +++ b/web/apps/photos/src/utils/machineLearning/faceCrop.ts @@ -10,7 +10,6 @@ import { import { cropWithRotation, imageBitmapToBlob } from "utils/image"; import { enlargeBox } from "."; import { Box } from "../../../thirdparty/face-api/classes"; -import { getAlignedFaceBox } from "./faceAlign"; export function getFaceCrop( imageBitmap: ImageBitmap, @@ -31,6 +30,15 @@ export function getFaceCrop( }; } +function getAlignedFaceBox(alignment: FaceAlignment) { + return new Box({ + x: alignment.center.x - alignment.size / 2, + y: alignment.center.y - alignment.size / 2, + width: alignment.size, + height: alignment.size, + }).round(); +} + export async function storeFaceCrop( faceId: string, faceCrop: FaceCrop, diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index 5b13b2faf..d0eacf1a6 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -25,15 +25,6 @@ export function newBox(x: number, y: number, width: number, height: number) { return new Box({ x, y, width, height }); } -export function newBoxFromPoints( - left: number, - top: number, - right: number, - bottom: number, -) { - return new Box({ left, top, right, bottom }); -} - export function getBoxCenterPt(topLeft: Point, bottomRight: Point): Point { return topLeft.add(bottomRight.sub(topLeft).div(new Point(2, 2))); } diff --git a/web/apps/photos/src/utils/machineLearning/transform.ts b/web/apps/photos/src/utils/machineLearning/transform.ts index 9e900bbe0..e69de29bb 100644 --- a/web/apps/photos/src/utils/machineLearning/transform.ts +++ b/web/apps/photos/src/utils/machineLearning/transform.ts @@ -1,33 +0,0 @@ -import { newBoxFromPoints } from "."; -import { Box, Point } from "../../../thirdparty/face-api/classes"; - -import { - Matrix, - applyToPoint, - compose, - scale, - translate, -} from "transformation-matrix"; - -export function computeTransformToBox(inBox: Box, toBox: Box): Matrix { - return compose( - translate(toBox.x, toBox.y), - scale(toBox.width / inBox.width, toBox.height / inBox.height), - ); -} - -export function transformPoint(point: Point, transform: Matrix) { - const txdPoint = applyToPoint(transform, point); - return new Point(txdPoint.x, txdPoint.y); -} - -export function transformPoints(points: Point[], transform: Matrix) { - return points?.map((p) => transformPoint(p, transform)); -} - -export function transformBox(box: Box, transform: Matrix) { - const topLeft = transformPoint(box.topLeft, transform); - const bottomRight = transformPoint(box.bottomRight, transform); - - return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y); -}