Inline

web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts

@@ -1,4 +1,11 @@
 import { MAX_FACE_DISTANCE_PERCENT } from "constants/mlConfig";
+import {
+    Matrix,
+    applyToPoint,
+    compose,
+    scale,
+    translate,
+} from "transformation-matrix";
 import { Dimensions } from "types/image";
 import {
     FaceDetection,
@@ -12,11 +19,6 @@ import {
     normalizePixelBetween0And1,
 } from "utils/image";
 import { newBox } from "utils/machineLearning";
-import {
-    computeTransformToBox,
-    transformBox,
-    transformPoints,
-} from "utils/machineLearning/transform";
 import { Box, Point } from "../../../thirdparty/face-api/classes";
 
 // TODO(MR): onnx-yolo
@@ -385,3 +387,35 @@ function getDetectionCenter(detection: FaceDetection) {
 
     return center.div({ x: 4, y: 4 });
 }
+
+function computeTransformToBox(inBox: Box, toBox: Box): Matrix {
+    return compose(
+        translate(toBox.x, toBox.y),
+        scale(toBox.width / inBox.width, toBox.height / inBox.height),
+    );
+}
+
+function transformPoint(point: Point, transform: Matrix) {
+    const txdPoint = applyToPoint(transform, point);
+    return new Point(txdPoint.x, txdPoint.y);
+}
+
+function transformPoints(points: Point[], transform: Matrix) {
+    return points?.map((p) => transformPoint(p, transform));
+}
+
+function transformBox(box: Box, transform: Matrix) {
+    const topLeft = transformPoint(box.topLeft, transform);
+    const bottomRight = transformPoint(box.bottomRight, transform);
+
+    return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y);
+}
+
+function newBoxFromPoints(
+    left: number,
+    top: number,
+    right: number,
+    bottom: number,
+) {
+    return new Box({ left, top, right, bottom });
+}

web/apps/photos/src/utils/machineLearning/faceAlign.ts

@@ -1,20 +1,39 @@
 import { Matrix } from "ml-matrix";
 import { getSimilarityTransformation } from "similarity-transformation";
-import { Dimensions } from "types/image";
 import { FaceAlignment, FaceDetection } from "types/machineLearning";
-import { cropWithRotation, transform } from "utils/image";
-import { Box, Point } from "../../../thirdparty/face-api/classes";
+import { Point } from "../../../thirdparty/face-api/classes";
 
-export function normalizeLandmarks(
-    landmarks: Array<[number, number]>,
-    faceSize: number,
-): Array<[number, number]> {
-    return landmarks.map((landmark) =>
-        landmark.map((p) => p / faceSize),
-    ) as Array<[number, number]>;
+const ARCFACE_LANDMARKS = [
+    [38.2946, 51.6963],
+    [73.5318, 51.5014],
+    [56.0252, 71.7366],
+    [56.1396, 92.2848],
+] as Array<[number, number]>;
+
+const ARCFACE_LANDMARKS_FACE_SIZE = 112;
+
+const ARC_FACE_5_LANDMARKS = [
+    [38.2946, 51.6963],
+    [73.5318, 51.5014],
+    [56.0252, 71.7366],
+    [41.5493, 92.3655],
+    [70.7299, 92.2041],
+] as Array<[number, number]>;
+
+export function getArcfaceAlignment(
+    faceDetection: FaceDetection,
+): FaceAlignment {
+    const landmarkCount = faceDetection.landmarks.length;
+    return getFaceAlignmentUsingSimilarityTransform(
+        faceDetection,
+        normalizeLandmarks(
+            landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS,
+            ARCFACE_LANDMARKS_FACE_SIZE,
+        ),
+    );
 }
 
-export function getFaceAlignmentUsingSimilarityTransform(
+function getFaceAlignmentUsingSimilarityTransform(
     faceDetection: FaceDetection,
     alignedLandmarks: Array<[number, number]>,
     // alignmentMethod: Versioned<FaceAlignmentMethod>
@@ -58,83 +77,11 @@ export function getFaceAlignmentUsingSimilarityTransform(
     };
 }
 
-const ARCFACE_LANDMARKS = [
-    [38.2946, 51.6963],
-    [73.5318, 51.5014],
-    [56.0252, 71.7366],
-    [56.1396, 92.2848],
-] as Array<[number, number]>;
-
-const ARCFACE_LANDMARKS_FACE_SIZE = 112;
-
-const ARC_FACE_5_LANDMARKS = [
-    [38.2946, 51.6963],
-    [73.5318, 51.5014],
-    [56.0252, 71.7366],
-    [41.5493, 92.3655],
-    [70.7299, 92.2041],
-] as Array<[number, number]>;
-
-export function getArcfaceAlignment(
-    faceDetection: FaceDetection,
-): FaceAlignment {
-    const landmarkCount = faceDetection.landmarks.length;
-    return getFaceAlignmentUsingSimilarityTransform(
-        faceDetection,
-        normalizeLandmarks(
-            landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS,
-            ARCFACE_LANDMARKS_FACE_SIZE,
-        ),
-    );
-}
-
-export function getAlignedFaceBox(alignment: FaceAlignment) {
-    return new Box({
-        x: alignment.center.x - alignment.size / 2,
-        y: alignment.center.y - alignment.size / 2,
-        width: alignment.size,
-        height: alignment.size,
-    }).round();
-}
-
-export function ibExtractFaceImage(
-    image: ImageBitmap,
-    alignment: FaceAlignment,
-    faceSize: number,
-): ImageBitmap {
-    const box = getAlignedFaceBox(alignment);
-    const faceSizeDimentions: Dimensions = {
-        width: faceSize,
-        height: faceSize,
-    };
-    return cropWithRotation(
-        image,
-        box,
-        alignment.rotation,
-        faceSizeDimentions,
-        faceSizeDimentions,
-    );
-}
-
-// Used in MLDebugViewOnly
-export function ibExtractFaceImageUsingTransform(
-    image: ImageBitmap,
-    alignment: FaceAlignment,
-    faceSize: number,
-): ImageBitmap {
-    const scaledMatrix = new Matrix(alignment.affineMatrix)
-        .mul(faceSize)
-        .to2DArray();
-    // log.info("scaledMatrix: ", scaledMatrix);
-    return transform(image, scaledMatrix, faceSize, faceSize);
-}
-
-export function ibExtractFaceImages(
-    image: ImageBitmap,
-    alignments: Array<FaceAlignment>,
+function normalizeLandmarks(
+    landmarks: Array<[number, number]>,
     faceSize: number,
-): Array<ImageBitmap> {
-    return alignments.map((alignment) =>
-        ibExtractFaceImage(image, alignment, faceSize),
-    );
+): Array<[number, number]> {
+    return landmarks.map((landmark) =>
+        landmark.map((p) => p / faceSize),
+    ) as Array<[number, number]>;
 }

web/apps/photos/src/utils/machineLearning/faceCrop.ts

@@ -10,7 +10,6 @@ import {
 import { cropWithRotation, imageBitmapToBlob } from "utils/image";
 import { enlargeBox } from ".";
 import { Box } from "../../../thirdparty/face-api/classes";
-import { getAlignedFaceBox } from "./faceAlign";
 
 export function getFaceCrop(
     imageBitmap: ImageBitmap,
@@ -31,6 +30,15 @@ export function getFaceCrop(
     };
 }
 
+function getAlignedFaceBox(alignment: FaceAlignment) {
+    return new Box({
+        x: alignment.center.x - alignment.size / 2,
+        y: alignment.center.y - alignment.size / 2,
+        width: alignment.size,
+        height: alignment.size,
+    }).round();
+}
+
 export async function storeFaceCrop(
     faceId: string,
     faceCrop: FaceCrop,

web/apps/photos/src/utils/machineLearning/index.ts

@@ -25,15 +25,6 @@ export function newBox(x: number, y: number, width: number, height: number) {
     return new Box({ x, y, width, height });
 }
 
-export function newBoxFromPoints(
-    left: number,
-    top: number,
-    right: number,
-    bottom: number,
-) {
-    return new Box({ left, top, right, bottom });
-}
-
 export function getBoxCenterPt(topLeft: Point, bottomRight: Point): Point {
     return topLeft.add(bottomRight.sub(topLeft).div(new Point(2, 2)));
 }

web/apps/photos/src/utils/machineLearning/transform.ts

@@ -1,33 +0,0 @@
-import { newBoxFromPoints } from ".";
-import { Box, Point } from "../../../thirdparty/face-api/classes";
-
-import {
-    Matrix,
-    applyToPoint,
-    compose,
-    scale,
-    translate,
-} from "transformation-matrix";
-
-export function computeTransformToBox(inBox: Box, toBox: Box): Matrix {
-    return compose(
-        translate(toBox.x, toBox.y),
-        scale(toBox.width / inBox.width, toBox.height / inBox.height),
-    );
-}
-
-export function transformPoint(point: Point, transform: Matrix) {
-    const txdPoint = applyToPoint(transform, point);
-    return new Point(txdPoint.x, txdPoint.y);
-}
-
-export function transformPoints(points: Point[], transform: Matrix) {
-    return points?.map((p) => transformPoint(p, transform));
-}
-
-export function transformBox(box: Box, transform: Matrix) {
-    const topLeft = transformPoint(box.topLeft, transform);
-    const bottomRight = transformPoint(box.bottomRight, transform);
-
-    return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y);
-}