0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Inline

Browse source
This commit is contained in:
Manav Rathi 2024-04-11 12:14:59 +05:30
parent 6ba5852876
commit e57e44c139
No known key found for this signature in database
5 changed files with 84 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts
View file

@ -1,4 +1,11 @@
import { MAX_FACE_DISTANCE_PERCENT } from "constants/mlConfig";
import {
Matrix,
applyToPoint,
compose,
scale,
translate,
} from "transformation-matrix";
import { Dimensions } from "types/image";
import {
FaceDetection,
@ -12,11 +19,6 @@ import {
normalizePixelBetween0And1,
} from "utils/image";
import { newBox } from "utils/machineLearning";
import {
computeTransformToBox,
transformBox,
transformPoints,
} from "utils/machineLearning/transform";
import { Box, Point } from "../../../thirdparty/face-api/classes";
// TODO(MR): onnx-yolo
@ -385,3 +387,35 @@ function getDetectionCenter(detection: FaceDetection) {
return center.div({ x: 4, y: 4 });
}
function computeTransformToBox(inBox: Box, toBox: Box): Matrix {
return compose(
translate(toBox.x, toBox.y),
scale(toBox.width / inBox.width, toBox.height / inBox.height),
);
}
function transformPoint(point: Point, transform: Matrix) {
const txdPoint = applyToPoint(transform, point);
return new Point(txdPoint.x, txdPoint.y);
}
function transformPoints(points: Point[], transform: Matrix) {
return points?.map((p) => transformPoint(p, transform));
}
function transformBox(box: Box, transform: Matrix) {
const topLeft = transformPoint(box.topLeft, transform);
const bottomRight = transformPoint(box.bottomRight, transform);
return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y);
}
function newBoxFromPoints(
left: number,
top: number,
right: number,
bottom: number,
) {
return new Box({ left, top, right, bottom });
}

125
web/apps/photos/src/utils/machineLearning/faceAlign.ts
View file

@ -1,20 +1,39 @@
import { Matrix } from "ml-matrix";
import { getSimilarityTransformation } from "similarity-transformation";
import { Dimensions } from "types/image";
import { FaceAlignment, FaceDetection } from "types/machineLearning";
import { cropWithRotation, transform } from "utils/image";
import { Box, Point } from "../../../thirdparty/face-api/classes";
import { Point } from "../../../thirdparty/face-api/classes";
export function normalizeLandmarks(
landmarks: Array<[number, number]>,
faceSize: number,
): Array<[number, number]> {
return landmarks.map((landmark) =>
landmark.map((p) => p / faceSize),
) as Array<[number, number]>;
const ARCFACE_LANDMARKS = [
[38.2946, 51.6963],
[73.5318, 51.5014],
[56.0252, 71.7366],
[56.1396, 92.2848],
] as Array<[number, number]>;
const ARCFACE_LANDMARKS_FACE_SIZE = 112;
const ARC_FACE_5_LANDMARKS = [
[38.2946, 51.6963],
[73.5318, 51.5014],
[56.0252, 71.7366],
[41.5493, 92.3655],
[70.7299, 92.2041],
] as Array<[number, number]>;
export function getArcfaceAlignment(
faceDetection: FaceDetection,
): FaceAlignment {
const landmarkCount = faceDetection.landmarks.length;
return getFaceAlignmentUsingSimilarityTransform(
faceDetection,
normalizeLandmarks(
landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS,
ARCFACE_LANDMARKS_FACE_SIZE,
),
);
}
export function getFaceAlignmentUsingSimilarityTransform(
function getFaceAlignmentUsingSimilarityTransform(
faceDetection: FaceDetection,
alignedLandmarks: Array<[number, number]>,
// alignmentMethod: Versioned<FaceAlignmentMethod>
@ -58,83 +77,11 @@ export function getFaceAlignmentUsingSimilarityTransform(
};
}
const ARCFACE_LANDMARKS = [
[38.2946, 51.6963],
[73.5318, 51.5014],
[56.0252, 71.7366],
[56.1396, 92.2848],
] as Array<[number, number]>;
const ARCFACE_LANDMARKS_FACE_SIZE = 112;
const ARC_FACE_5_LANDMARKS = [
[38.2946, 51.6963],
[73.5318, 51.5014],
[56.0252, 71.7366],
[41.5493, 92.3655],
[70.7299, 92.2041],
] as Array<[number, number]>;
export function getArcfaceAlignment(
faceDetection: FaceDetection,
): FaceAlignment {
const landmarkCount = faceDetection.landmarks.length;
return getFaceAlignmentUsingSimilarityTransform(
faceDetection,
normalizeLandmarks(
landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS,
ARCFACE_LANDMARKS_FACE_SIZE,
),
);
}
export function getAlignedFaceBox(alignment: FaceAlignment) {
return new Box({
x: alignment.center.x - alignment.size / 2,
y: alignment.center.y - alignment.size / 2,
width: alignment.size,
height: alignment.size,
}).round();
}
export function ibExtractFaceImage(
image: ImageBitmap,
alignment: FaceAlignment,
function normalizeLandmarks(
landmarks: Array<[number, number]>,
faceSize: number,
): ImageBitmap {
const box = getAlignedFaceBox(alignment);
const faceSizeDimentions: Dimensions = {
width: faceSize,
height: faceSize,
};
return cropWithRotation(
image,
box,
alignment.rotation,
faceSizeDimentions,
faceSizeDimentions,
);
}
// Used in MLDebugViewOnly
export function ibExtractFaceImageUsingTransform(
image: ImageBitmap,
alignment: FaceAlignment,
faceSize: number,
): ImageBitmap {
const scaledMatrix = new Matrix(alignment.affineMatrix)
.mul(faceSize)
.to2DArray();
// log.info("scaledMatrix: ", scaledMatrix);
return transform(image, scaledMatrix, faceSize, faceSize);
}
export function ibExtractFaceImages(
image: ImageBitmap,
alignments: Array<FaceAlignment>,
faceSize: number,
): Array<ImageBitmap> {
return alignments.map((alignment) =>
ibExtractFaceImage(image, alignment, faceSize),
);
): Array<[number, number]> {
return landmarks.map((landmark) =>
landmark.map((p) => p / faceSize),
) as Array<[number, number]>;
}

10
web/apps/photos/src/utils/machineLearning/faceCrop.ts
View file

@ -10,7 +10,6 @@ import {
import { cropWithRotation, imageBitmapToBlob } from "utils/image";
import { enlargeBox } from ".";
import { Box } from "../../../thirdparty/face-api/classes";
import { getAlignedFaceBox } from "./faceAlign";
export function getFaceCrop(
imageBitmap: ImageBitmap,
@ -31,6 +30,15 @@ export function getFaceCrop(
};
}
function getAlignedFaceBox(alignment: FaceAlignment) {
return new Box({
x: alignment.center.x - alignment.size / 2,
y: alignment.center.y - alignment.size / 2,
width: alignment.size,
height: alignment.size,
}).round();
}
export async function storeFaceCrop(
faceId: string,
faceCrop: FaceCrop,

9
web/apps/photos/src/utils/machineLearning/index.ts
View file

@ -25,15 +25,6 @@ export function newBox(x: number, y: number, width: number, height: number) {
return new Box({ x, y, width, height });
}
export function newBoxFromPoints(
left: number,
top: number,
right: number,
bottom: number,
) {
return new Box({ left, top, right, bottom });
}
export function getBoxCenterPt(topLeft: Point, bottomRight: Point): Point {
return topLeft.add(bottomRight.sub(topLeft).div(new Point(2, 2)));
}

33
web/apps/photos/src/utils/machineLearning/transform.ts
View file

@ -1,33 +0,0 @@
import { newBoxFromPoints } from ".";
import { Box, Point } from "../../../thirdparty/face-api/classes";
import {
Matrix,
applyToPoint,
compose,
scale,
translate,
} from "transformation-matrix";
export function computeTransformToBox(inBox: Box, toBox: Box): Matrix {
return compose(
translate(toBox.x, toBox.y),
scale(toBox.width / inBox.width, toBox.height / inBox.height),
);
}
export function transformPoint(point: Point, transform: Matrix) {
const txdPoint = applyToPoint(transform, point);
return new Point(txdPoint.x, txdPoint.y);
}
export function transformPoints(points: Point[], transform: Matrix) {
return points?.map((p) => transformPoint(p, transform));
}
export function transformBox(box: Box, transform: Matrix) {
const topLeft = transformPoint(box.topLeft, transform);
const bottomRight = transformPoint(box.bottomRight, transform);
return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y);
}