[web] Remove tensorflow, and then some (#1411)
Remove all ML code that is not in the immediate release path. We will resurrect if/when we reintroduce object and text detection, for now we prune in an attempt to get the face detection release out to completion.
This commit is contained in:
Manav Rathi
GitHub
commit
bbdbad5891
68 changed files with 200 additions and 16853 deletions
|
|
@ -10,14 +10,7 @@
|
|||
"@ente/shared": "*",
|
||||
"@mui/x-date-pickers": "^5.0.0-alpha.6",
|
||||
"@stripe/stripe-js": "^1.13.2",
|
||||
"@tensorflow-models/coco-ssd": "^2.2.2",
|
||||
"@tensorflow/tfjs-backend-cpu": "^4.10.0",
|
||||
"@tensorflow/tfjs-backend-webgl": "^4.9.0",
|
||||
"@tensorflow/tfjs-converter": "^4.10.0",
|
||||
"@tensorflow/tfjs-core": "^4.10.0",
|
||||
"@tensorflow/tfjs-tflite": "0.0.1-alpha.7",
|
||||
"bip39": "^3.0.4",
|
||||
"blazeface-back": "^0.0.9",
|
||||
"bs58": "^5.0.0",
|
||||
"chrono-node": "^2.2.6",
|
||||
"date-fns": "^2",
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
"use strict";var Module={};var initializedJS=false;function threadPrintErr(){var text=Array.prototype.slice.call(arguments).join(" ");console.error(text)}function threadAlert(){var text=Array.prototype.slice.call(arguments).join(" ");postMessage({cmd:"alert",text:text,threadId:Module["_pthread_self"]()})}var err=threadPrintErr;self.alert=threadAlert;Module["instantiateWasm"]=function(info,receiveInstance){var instance=new WebAssembly.Instance(Module["wasmModule"],info);receiveInstance(instance);Module["wasmModule"]=null;return instance.exports};function moduleLoaded(){}self.onmessage=function(e){try{if(e.data.cmd==="load"){Module["wasmModule"]=e.data.wasmModule;Module["wasmMemory"]=e.data.wasmMemory;Module["buffer"]=Module["wasmMemory"].buffer;Module["ENVIRONMENT_IS_PTHREAD"]=true;if(typeof e.data.urlOrBlob==="string"){importScripts(e.data.urlOrBlob)}else{var objectUrl=URL.createObjectURL(e.data.urlOrBlob);importScripts(objectUrl);URL.revokeObjectURL(objectUrl)}tflite_web_api_ModuleFactory(Module).then(function(instance){Module=instance;moduleLoaded()})}else if(e.data.cmd==="objectTransfer"){Module["PThread"].receiveObjectTransfer(e.data)}else if(e.data.cmd==="run"){Module["__performance_now_clock_drift"]=performance.now()-e.data.time;Module["__emscripten_thread_init"](e.data.threadInfoStruct,0,0);var max=e.data.stackBase;var top=e.data.stackBase+e.data.stackSize;Module["establishStackSpace"](top,max);Module["PThread"].receiveObjectTransfer(e.data);Module["PThread"].threadInit();if(!initializedJS){Module["___embind_register_native_and_builtin_types"]();initializedJS=true}try{var result=Module["invokeEntryPoint"](e.data.start_routine,e.data.arg);if(Module["keepRuntimeAlive"]()){Module["PThread"].setExitStatus(result)}else{Module["PThread"].threadExit(result)}}catch(ex){if(ex==="Canceled!"){Module["PThread"].threadCancel()}else if(ex!="unwind"){if(ex instanceof Module["ExitStatus"]){if(Module["keepRuntimeAlive"]()){}else{Module["PThread"].threadExit(ex.status)}}else{Module["PThread"].threadExit(-2);throw ex}}}}else if(e.data.cmd==="cancel"){if(Module["_pthread_self"]()){Module["PThread"].threadCancel()}}else if(e.data.target==="setimmediate"){}else if(e.data.cmd==="processThreadQueue"){if(Module["_pthread_self"]()){Module["_emscripten_current_thread_process_queued_calls"]()}}else{err("worker.js received unknown command "+e.data.cmd);err(e.data)}}catch(ex){err("worker.js onmessage() captured an uncaught exception: "+ex);if(ex&&ex.stack)err(ex.stack);throw ex}};
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
"use strict";var Module={};var initializedJS=false;function threadPrintErr(){var text=Array.prototype.slice.call(arguments).join(" ");console.error(text)}function threadAlert(){var text=Array.prototype.slice.call(arguments).join(" ");postMessage({cmd:"alert",text:text,threadId:Module["_pthread_self"]()})}var err=threadPrintErr;self.alert=threadAlert;Module["instantiateWasm"]=function(info,receiveInstance){var instance=new WebAssembly.Instance(Module["wasmModule"],info);receiveInstance(instance);Module["wasmModule"]=null;return instance.exports};function moduleLoaded(){}self.onmessage=function(e){try{if(e.data.cmd==="load"){Module["wasmModule"]=e.data.wasmModule;Module["wasmMemory"]=e.data.wasmMemory;Module["buffer"]=Module["wasmMemory"].buffer;Module["ENVIRONMENT_IS_PTHREAD"]=true;if(typeof e.data.urlOrBlob==="string"){importScripts(e.data.urlOrBlob)}else{var objectUrl=URL.createObjectURL(e.data.urlOrBlob);importScripts(objectUrl);URL.revokeObjectURL(objectUrl)}tflite_web_api_ModuleFactory(Module).then(function(instance){Module=instance;moduleLoaded()})}else if(e.data.cmd==="objectTransfer"){Module["PThread"].receiveObjectTransfer(e.data)}else if(e.data.cmd==="run"){Module["__performance_now_clock_drift"]=performance.now()-e.data.time;Module["__emscripten_thread_init"](e.data.threadInfoStruct,0,0);var max=e.data.stackBase;var top=e.data.stackBase+e.data.stackSize;Module["establishStackSpace"](top,max);Module["PThread"].receiveObjectTransfer(e.data);Module["PThread"].threadInit();if(!initializedJS){Module["___embind_register_native_and_builtin_types"]();initializedJS=true}try{var result=Module["invokeEntryPoint"](e.data.start_routine,e.data.arg);if(Module["keepRuntimeAlive"]()){Module["PThread"].setExitStatus(result)}else{Module["PThread"].threadExit(result)}}catch(ex){if(ex==="Canceled!"){Module["PThread"].threadCancel()}else if(ex!="unwind"){if(ex instanceof Module["ExitStatus"]){if(Module["keepRuntimeAlive"]()){}else{Module["PThread"].threadExit(ex.status)}}else{Module["PThread"].threadExit(-2);throw ex}}}}else if(e.data.cmd==="cancel"){if(Module["_pthread_self"]()){Module["PThread"].threadCancel()}}else if(e.data.target==="setimmediate"){}else if(e.data.cmd==="processThreadQueue"){if(Module["_pthread_self"]()){Module["_emscripten_current_thread_process_queued_calls"]()}}else{err("worker.js received unknown command "+e.data.cmd);err(e.data)}}catch(ex){err("worker.js onmessage() captured an uncaught exception: "+ex);if(ex&&ex.stack)err(ex.stack);throw ex}};
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
|
|
@ -1,32 +0,0 @@
|
|||
{
|
||||
"0": "waterfall",
|
||||
"1": "snow",
|
||||
"2": "landscape",
|
||||
"3": "underwater",
|
||||
"4": "architecture",
|
||||
"5": "sunset / sunrise",
|
||||
"6": "blue sky",
|
||||
"7": "cloudy sky",
|
||||
"8": "greenery",
|
||||
"9": "autumn leaves",
|
||||
"10": "portrait",
|
||||
"11": "flower",
|
||||
"12": "night shot",
|
||||
"13": "stage concert",
|
||||
"14": "fireworks",
|
||||
"15": "candle light",
|
||||
"16": "neon lights",
|
||||
"17": "indoor",
|
||||
"18": "backlight",
|
||||
"19": "text documents",
|
||||
"20": "qr images",
|
||||
"21": "group portrait",
|
||||
"22": "computer screens",
|
||||
"23": "kids",
|
||||
"24": "dog",
|
||||
"25": "cat",
|
||||
"26": "macro",
|
||||
"27": "food",
|
||||
"28": "beach",
|
||||
"29": "mountain"
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load diff
File diff suppressed because one or more lines are too long
|
|
@ -1,51 +0,0 @@
|
|||
import Box from "@mui/material/Box";
|
||||
import { Chip } from "components/Chip";
|
||||
import { Legend } from "components/PhotoViewer/styledComponents/Legend";
|
||||
import { t } from "i18next";
|
||||
import { useEffect, useState } from "react";
|
||||
import { EnteFile } from "types/file";
|
||||
import mlIDbStorage from "utils/storage/mlIDbStorage";
|
||||
|
||||
export function ObjectLabelList(props: {
|
||||
file: EnteFile;
|
||||
updateMLDataIndex: number;
|
||||
}) {
|
||||
const [objects, setObjects] = useState<Array<string>>([]);
|
||||
useEffect(() => {
|
||||
let didCancel = false;
|
||||
const main = async () => {
|
||||
const objects = await mlIDbStorage.getAllObjectsMap();
|
||||
const uniqueObjectNames = [
|
||||
...new Set(
|
||||
(objects.get(props.file.id) ?? []).map(
|
||||
(object) => object.detection.class,
|
||||
),
|
||||
),
|
||||
];
|
||||
!didCancel && setObjects(uniqueObjectNames);
|
||||
};
|
||||
main();
|
||||
return () => {
|
||||
didCancel = true;
|
||||
};
|
||||
}, [props.file, props.updateMLDataIndex]);
|
||||
|
||||
if (objects.length === 0) return <></>;
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Legend sx={{ pb: 1, display: "block" }}>{t("OBJECTS")}</Legend>
|
||||
<Box
|
||||
display={"flex"}
|
||||
gap={1}
|
||||
flexWrap="wrap"
|
||||
justifyContent={"flex-start"}
|
||||
alignItems={"flex-start"}
|
||||
>
|
||||
{objects.map((object) => (
|
||||
<Chip key={object}>{object}</Chip>
|
||||
))}
|
||||
</Box>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import { useEffect, useRef } from "react";
|
||||
import { FaceImage } from "types/machineLearning";
|
||||
|
||||
interface FaceImageProps {
|
||||
faceImage: FaceImage;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
|
||||
export default function TFJSImage(props: FaceImageProps) {
|
||||
const canvasRef = useRef(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!props || !props.faceImage) {
|
||||
return;
|
||||
}
|
||||
const canvas = canvasRef.current;
|
||||
const faceTensor = tf.tensor3d(props.faceImage);
|
||||
const resized =
|
||||
props.width && props.height
|
||||
? tf.image.resizeBilinear(faceTensor, [
|
||||
props.width,
|
||||
props.height,
|
||||
])
|
||||
: faceTensor;
|
||||
const normFaceImage = tf.div(tf.add(resized, 1.0), 2);
|
||||
tf.browser.toPixels(normFaceImage as tf.Tensor3D, canvas);
|
||||
}, [props]);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
width={112}
|
||||
height={112}
|
||||
style={{ display: "inline" }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
@ -10,7 +10,6 @@ import TextSnippetOutlined from "@mui/icons-material/TextSnippetOutlined";
|
|||
import { Box, DialogProps, Link, Stack, styled } from "@mui/material";
|
||||
import { Chip } from "components/Chip";
|
||||
import { EnteDrawer } from "components/EnteDrawer";
|
||||
import { ObjectLabelList } from "components/MachineLearning/ObjectList";
|
||||
import {
|
||||
PhotoPeopleList,
|
||||
UnidentifiedFaces,
|
||||
|
|
@ -344,10 +343,6 @@ export function FileInfo({
|
|||
file={file}
|
||||
updateMLDataIndex={updateMLDataIndex}
|
||||
/>
|
||||
<ObjectLabelList
|
||||
file={file}
|
||||
updateMLDataIndex={updateMLDataIndex}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</Stack>
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import {
|
|||
import { Collection } from "types/collection";
|
||||
import { LocationTagData } from "types/entity";
|
||||
import { EnteFile } from "types/file";
|
||||
import { Person, Thing, WordGroup } from "types/machineLearning";
|
||||
import { Person } from "types/machineLearning";
|
||||
import {
|
||||
ClipSearchScores,
|
||||
DateValue,
|
||||
|
|
@ -146,12 +146,6 @@ export default function SearchInput(props: Iprops) {
|
|||
case SuggestionType.PERSON:
|
||||
search = { person: selectedOption.value as Person };
|
||||
break;
|
||||
case SuggestionType.THING:
|
||||
search = { thing: selectedOption.value as Thing };
|
||||
break;
|
||||
case SuggestionType.TEXT:
|
||||
search = { text: selectedOption.value as WordGroup };
|
||||
break;
|
||||
case SuggestionType.FILE_TYPE:
|
||||
search = { fileType: selectedOption.value as FILE_TYPE };
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -46,22 +46,6 @@ export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = {
|
|||
// maxDistanceInsideCluster: 0.4,
|
||||
generateDebugInfo: true,
|
||||
},
|
||||
objectDetection: {
|
||||
method: "SSDMobileNetV2",
|
||||
maxNumBoxes: 20,
|
||||
minScore: 0.2,
|
||||
},
|
||||
sceneDetection: {
|
||||
method: "ImageScene",
|
||||
minScore: 0.1,
|
||||
},
|
||||
// tsne: {
|
||||
// samples: 200,
|
||||
// dim: 2,
|
||||
// perplexity: 10.0,
|
||||
// learningRate: 10.0,
|
||||
// metric: 'euclidean',
|
||||
// },
|
||||
mlVersion: 3,
|
||||
};
|
||||
|
||||
|
|
@ -77,21 +61,7 @@ export const MAX_ML_SYNC_ERROR_COUNT = 1;
|
|||
|
||||
export const TEXT_DETECTION_TIMEOUT_MS = [10000, 30000, 60000, 120000, 240000];
|
||||
|
||||
export const BLAZEFACE_MAX_FACES = 50;
|
||||
export const BLAZEFACE_INPUT_SIZE = 256;
|
||||
export const BLAZEFACE_IOU_THRESHOLD = 0.3;
|
||||
export const BLAZEFACE_SCORE_THRESHOLD = 0.75;
|
||||
export const BLAZEFACE_PASS1_SCORE_THRESHOLD = 0.4;
|
||||
export const BLAZEFACE_FACE_SIZE = 112;
|
||||
export const MOBILEFACENET_FACE_SIZE = 112;
|
||||
export const MOBILEFACENET_EMBEDDING_SIZE = 192;
|
||||
|
||||
// scene detection model takes fixed-shaped (224x224) inputs
|
||||
// https://tfhub.dev/sayannath/lite-model/image-scene/1
|
||||
export const SCENE_DETECTION_IMAGE_SIZE = 224;
|
||||
|
||||
// SSD with Mobilenet v2 initialized from Imagenet classification checkpoint. Trained on COCO 2017 dataset (images scaled to 320x320 resolution).
|
||||
// https://tfhub.dev/tensorflow/ssd_mobilenet_v2/2
|
||||
export const OBJECT_DETECTION_IMAGE_SIZE = 320;
|
||||
|
||||
export const BATCHES_BEFORE_SYNCING_INDEX = 5;
|
||||
|
|
|
|||
|
|
@ -1,257 +0,0 @@
|
|||
import log from "@/next/log";
|
||||
import { GraphModel } from "@tensorflow/tfjs-converter";
|
||||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import {
|
||||
load as blazeFaceLoad,
|
||||
BlazeFaceModel,
|
||||
NormalizedFace,
|
||||
} from "blazeface-back";
|
||||
import {
|
||||
BLAZEFACE_FACE_SIZE,
|
||||
BLAZEFACE_INPUT_SIZE,
|
||||
BLAZEFACE_IOU_THRESHOLD,
|
||||
BLAZEFACE_MAX_FACES,
|
||||
BLAZEFACE_PASS1_SCORE_THRESHOLD,
|
||||
BLAZEFACE_SCORE_THRESHOLD,
|
||||
MAX_FACE_DISTANCE_PERCENT,
|
||||
} from "constants/mlConfig";
|
||||
import {
|
||||
FaceDetection,
|
||||
FaceDetectionMethod,
|
||||
FaceDetectionService,
|
||||
Versioned,
|
||||
} from "types/machineLearning";
|
||||
import { addPadding, crop, resizeToSquare } from "utils/image";
|
||||
import { enlargeBox, newBox, normFaceBox } from "utils/machineLearning";
|
||||
import {
|
||||
getNearestDetection,
|
||||
removeDuplicateDetections,
|
||||
transformPaddedToImage,
|
||||
} from "utils/machineLearning/faceDetection";
|
||||
import {
|
||||
computeTransformToBox,
|
||||
transformBox,
|
||||
transformPoints,
|
||||
} from "utils/machineLearning/transform";
|
||||
import { Box, Point } from "../../../thirdparty/face-api/classes";
|
||||
|
||||
class BlazeFaceDetectionService implements FaceDetectionService {
|
||||
private blazeFaceModel: Promise<BlazeFaceModel>;
|
||||
private blazeFaceBackModel: GraphModel;
|
||||
public method: Versioned<FaceDetectionMethod>;
|
||||
|
||||
private desiredLeftEye = [0.36, 0.45];
|
||||
private desiredFaceSize;
|
||||
|
||||
public constructor(desiredFaceSize: number = BLAZEFACE_FACE_SIZE) {
|
||||
this.method = {
|
||||
value: "BlazeFace",
|
||||
version: 1,
|
||||
};
|
||||
this.desiredFaceSize = desiredFaceSize;
|
||||
}
|
||||
|
||||
public getRelativeDetection(): FaceDetection {
|
||||
// TODO(MR): onnx-yolo
|
||||
throw new Error();
|
||||
}
|
||||
|
||||
private async init() {
|
||||
this.blazeFaceModel = blazeFaceLoad({
|
||||
maxFaces: BLAZEFACE_MAX_FACES,
|
||||
scoreThreshold: BLAZEFACE_PASS1_SCORE_THRESHOLD,
|
||||
iouThreshold: BLAZEFACE_IOU_THRESHOLD,
|
||||
modelUrl: "/models/blazeface/back/model.json",
|
||||
inputHeight: BLAZEFACE_INPUT_SIZE,
|
||||
inputWidth: BLAZEFACE_INPUT_SIZE,
|
||||
});
|
||||
log.info(
|
||||
"loaded blazeFaceModel: ",
|
||||
// await this.blazeFaceModel,
|
||||
// eslint-disable-next-line @typescript-eslint/await-thenable
|
||||
await tf.getBackend(),
|
||||
);
|
||||
}
|
||||
|
||||
private getDlibAlignedFace(normFace: NormalizedFace): Box {
|
||||
const relX = 0.5;
|
||||
const relY = 0.43;
|
||||
const relScale = 0.45;
|
||||
|
||||
const leftEyeCenter = normFace.landmarks[0];
|
||||
const rightEyeCenter = normFace.landmarks[1];
|
||||
const mountCenter = normFace.landmarks[3];
|
||||
|
||||
const distToMouth = (pt) => {
|
||||
const dy = mountCenter[1] - pt[1];
|
||||
const dx = mountCenter[0] - pt[0];
|
||||
return Math.sqrt(dx * dx + dy * dy);
|
||||
};
|
||||
const eyeToMouthDist =
|
||||
(distToMouth(leftEyeCenter) + distToMouth(rightEyeCenter)) / 2;
|
||||
|
||||
const size = Math.floor(eyeToMouthDist / relScale);
|
||||
|
||||
const center = [
|
||||
(leftEyeCenter[0] + rightEyeCenter[0] + mountCenter[0]) / 3,
|
||||
(leftEyeCenter[1] + rightEyeCenter[1] + mountCenter[1]) / 3,
|
||||
];
|
||||
|
||||
const left = center[0] - relX * size;
|
||||
const top = center[1] - relY * size;
|
||||
const right = center[0] + relX * size;
|
||||
const bottom = center[1] + relY * size;
|
||||
|
||||
return new Box({
|
||||
left: left,
|
||||
top: top,
|
||||
right: right,
|
||||
bottom: bottom,
|
||||
});
|
||||
}
|
||||
|
||||
private getAlignedFace(normFace: NormalizedFace): Box {
|
||||
const leftEye = normFace.landmarks[0];
|
||||
const rightEye = normFace.landmarks[1];
|
||||
// const noseTip = normFace.landmarks[2];
|
||||
|
||||
const dy = rightEye[1] - leftEye[1];
|
||||
const dx = rightEye[0] - leftEye[0];
|
||||
|
||||
const desiredRightEyeX = 1.0 - this.desiredLeftEye[0];
|
||||
|
||||
// const eyesCenterX = (leftEye[0] + rightEye[0]) / 2;
|
||||
// const yaw = Math.abs(noseTip[0] - eyesCenterX)
|
||||
const dist = Math.sqrt(dx * dx + dy * dy);
|
||||
let desiredDist = desiredRightEyeX - this.desiredLeftEye[0];
|
||||
desiredDist *= this.desiredFaceSize;
|
||||
const scale = desiredDist / dist;
|
||||
// log.info("scale: ", scale);
|
||||
|
||||
const eyesCenter = [];
|
||||
eyesCenter[0] = Math.floor((leftEye[0] + rightEye[0]) / 2);
|
||||
eyesCenter[1] = Math.floor((leftEye[1] + rightEye[1]) / 2);
|
||||
// log.info("eyesCenter: ", eyesCenter);
|
||||
|
||||
const faceWidth = this.desiredFaceSize / scale;
|
||||
const faceHeight = this.desiredFaceSize / scale;
|
||||
// log.info("faceWidth: ", faceWidth, "faceHeight: ", faceHeight)
|
||||
|
||||
const tx = eyesCenter[0] - faceWidth * 0.5;
|
||||
const ty = eyesCenter[1] - faceHeight * this.desiredLeftEye[1];
|
||||
// log.info("tx: ", tx, "ty: ", ty);
|
||||
|
||||
return new Box({
|
||||
left: tx,
|
||||
top: ty,
|
||||
right: tx + faceWidth,
|
||||
bottom: ty + faceHeight,
|
||||
});
|
||||
}
|
||||
|
||||
public async detectFacesUsingModel(image: tf.Tensor3D) {
|
||||
const resizedImage = tf.image.resizeBilinear(image, [256, 256]);
|
||||
const reshapedImage = tf.reshape(resizedImage, [
|
||||
1,
|
||||
resizedImage.shape[0],
|
||||
resizedImage.shape[1],
|
||||
3,
|
||||
]);
|
||||
const normalizedImage = tf.sub(tf.div(reshapedImage, 127.5), 1.0);
|
||||
// eslint-disable-next-line @typescript-eslint/await-thenable
|
||||
const results = await this.blazeFaceBackModel.predict(normalizedImage);
|
||||
// log.info('onFacesDetected: ', results);
|
||||
return results;
|
||||
}
|
||||
|
||||
private async getBlazefaceModel() {
|
||||
if (!this.blazeFaceModel) {
|
||||
await this.init();
|
||||
}
|
||||
|
||||
return this.blazeFaceModel;
|
||||
}
|
||||
|
||||
private async estimateFaces(
|
||||
imageBitmap: ImageBitmap,
|
||||
): Promise<Array<FaceDetection>> {
|
||||
const resized = resizeToSquare(imageBitmap, BLAZEFACE_INPUT_SIZE);
|
||||
const tfImage = tf.browser.fromPixels(resized.image);
|
||||
const blazeFaceModel = await this.getBlazefaceModel();
|
||||
// TODO: check if this works concurrently, else use serialqueue
|
||||
const faces = await blazeFaceModel.estimateFaces(tfImage);
|
||||
tf.dispose(tfImage);
|
||||
|
||||
const inBox = newBox(0, 0, resized.width, resized.height);
|
||||
const toBox = newBox(0, 0, imageBitmap.width, imageBitmap.height);
|
||||
const transform = computeTransformToBox(inBox, toBox);
|
||||
// log.info("1st pass: ", { transform });
|
||||
|
||||
const faceDetections: Array<FaceDetection> = faces?.map((f) => {
|
||||
const box = transformBox(normFaceBox(f), transform);
|
||||
const normLandmarks = (f.landmarks as number[][])?.map(
|
||||
(l) => new Point(l[0], l[1]),
|
||||
);
|
||||
const landmarks = transformPoints(normLandmarks, transform);
|
||||
return {
|
||||
box,
|
||||
landmarks,
|
||||
probability: f.probability as number,
|
||||
// detectionMethod: this.method,
|
||||
} as FaceDetection;
|
||||
});
|
||||
|
||||
return faceDetections;
|
||||
}
|
||||
|
||||
public async detectFaces(
|
||||
imageBitmap: ImageBitmap,
|
||||
): Promise<Array<FaceDetection>> {
|
||||
const maxFaceDistance = imageBitmap.width * MAX_FACE_DISTANCE_PERCENT;
|
||||
const pass1Detections = await this.estimateFaces(imageBitmap);
|
||||
|
||||
// run 2nd pass for accuracy
|
||||
const detections: Array<FaceDetection> = [];
|
||||
for (const pass1Detection of pass1Detections) {
|
||||
const imageBox = enlargeBox(pass1Detection.box, 2);
|
||||
const faceImage = crop(
|
||||
imageBitmap,
|
||||
imageBox,
|
||||
BLAZEFACE_INPUT_SIZE / 2,
|
||||
);
|
||||
const paddedImage = addPadding(faceImage, 0.5);
|
||||
const paddedBox = enlargeBox(imageBox, 2);
|
||||
const pass2Detections = await this.estimateFaces(paddedImage);
|
||||
|
||||
pass2Detections?.forEach((d) =>
|
||||
transformPaddedToImage(d, faceImage, imageBox, paddedBox),
|
||||
);
|
||||
let selected = pass2Detections?.[0];
|
||||
if (pass2Detections?.length > 1) {
|
||||
// log.info('2nd pass >1 face', pass2Detections.length);
|
||||
selected = getNearestDetection(
|
||||
pass1Detection,
|
||||
pass2Detections,
|
||||
// maxFaceDistance
|
||||
);
|
||||
}
|
||||
|
||||
// we might miss 1st pass face actually having score within threshold
|
||||
// it is ok as results will be consistent with 2nd pass only detections
|
||||
if (selected && selected.probability >= BLAZEFACE_SCORE_THRESHOLD) {
|
||||
// log.info("pass2: ", { imageBox, paddedBox, transform, selected });
|
||||
detections.push(selected);
|
||||
}
|
||||
}
|
||||
|
||||
return removeDuplicateDetections(detections, maxFaceDistance);
|
||||
}
|
||||
|
||||
public async dispose() {
|
||||
const blazeFaceModel = await this.getBlazefaceModel();
|
||||
blazeFaceModel?.dispose();
|
||||
this.blazeFaceModel = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export default new BlazeFaceDetectionService();
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
import log from "@/next/log";
|
||||
import * as tfjsConverter from "@tensorflow/tfjs-converter";
|
||||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import { SCENE_DETECTION_IMAGE_SIZE } from "constants/mlConfig";
|
||||
import {
|
||||
ObjectDetection,
|
||||
SceneDetectionMethod,
|
||||
SceneDetectionService,
|
||||
Versioned,
|
||||
} from "types/machineLearning";
|
||||
import { resizeToSquare } from "utils/image";
|
||||
|
||||
class ImageScene implements SceneDetectionService {
|
||||
method: Versioned<SceneDetectionMethod>;
|
||||
private model: tfjsConverter.GraphModel;
|
||||
private sceneMap: { [key: string]: string };
|
||||
private ready: Promise<void>;
|
||||
private workerID: number;
|
||||
|
||||
public constructor() {
|
||||
this.method = {
|
||||
value: "ImageScene",
|
||||
version: 1,
|
||||
};
|
||||
this.workerID = Math.round(Math.random() * 1000);
|
||||
}
|
||||
|
||||
private async init() {
|
||||
log.info(`[${this.workerID}]`, "ImageScene init called");
|
||||
if (this.model) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.sceneMap = await (
|
||||
await fetch("/models/imagescene/sceneMap.json")
|
||||
).json();
|
||||
|
||||
this.model = await tfjsConverter.loadGraphModel(
|
||||
"/models/imagescene/model.json",
|
||||
);
|
||||
log.info(
|
||||
`[${this.workerID}]`,
|
||||
"loaded ImageScene model",
|
||||
tf.getBackend(),
|
||||
);
|
||||
|
||||
tf.tidy(() => {
|
||||
const zeroTensor = tf.zeros([1, 224, 224, 3]);
|
||||
// warmup the model
|
||||
this.model.predict(zeroTensor) as tf.Tensor;
|
||||
});
|
||||
}
|
||||
|
||||
private async getImageSceneModel() {
|
||||
log.info(`[${this.workerID}]`, "ImageScene getImageSceneModel called");
|
||||
if (!this.ready) {
|
||||
this.ready = this.init();
|
||||
}
|
||||
await this.ready;
|
||||
return this.model;
|
||||
}
|
||||
|
||||
async detectScenes(image: ImageBitmap, minScore: number) {
|
||||
const resized = resizeToSquare(image, SCENE_DETECTION_IMAGE_SIZE);
|
||||
|
||||
const model = await this.getImageSceneModel();
|
||||
|
||||
const output = tf.tidy(() => {
|
||||
const tfImage = tf.browser.fromPixels(resized.image);
|
||||
const input = tf.expandDims(tf.cast(tfImage, "float32"));
|
||||
const output = model.predict(input) as tf.Tensor;
|
||||
return output;
|
||||
});
|
||||
|
||||
const data = (await output.data()) as Float32Array;
|
||||
output.dispose();
|
||||
|
||||
const scenes = this.parseSceneDetectionResult(
|
||||
data,
|
||||
minScore,
|
||||
image.width,
|
||||
image.height,
|
||||
);
|
||||
|
||||
return scenes;
|
||||
}
|
||||
|
||||
private parseSceneDetectionResult(
|
||||
outputData: Float32Array,
|
||||
minScore: number,
|
||||
width: number,
|
||||
height: number,
|
||||
): ObjectDetection[] {
|
||||
const scenes = [];
|
||||
for (let i = 0; i < outputData.length; i++) {
|
||||
if (outputData[i] >= minScore) {
|
||||
scenes.push({
|
||||
class: this.sceneMap[i.toString()],
|
||||
score: outputData[i],
|
||||
bbox: [0, 0, width, height],
|
||||
});
|
||||
}
|
||||
}
|
||||
return scenes;
|
||||
}
|
||||
}
|
||||
|
||||
export default new ImageScene();
|
||||
|
|
@ -22,20 +22,14 @@ import {
|
|||
MLLibraryData,
|
||||
MLSyncConfig,
|
||||
MLSyncContext,
|
||||
ObjectDetectionMethod,
|
||||
ObjectDetectionService,
|
||||
SceneDetectionMethod,
|
||||
SceneDetectionService,
|
||||
} from "types/machineLearning";
|
||||
import { logQueueStats } from "utils/machineLearning";
|
||||
import arcfaceAlignmentService from "./arcfaceAlignmentService";
|
||||
import arcfaceCropService from "./arcfaceCropService";
|
||||
import dbscanClusteringService from "./dbscanClusteringService";
|
||||
import hdbscanClusteringService from "./hdbscanClusteringService";
|
||||
import imageSceneService from "./imageSceneService";
|
||||
import laplacianBlurDetectionService from "./laplacianBlurDetectionService";
|
||||
import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService";
|
||||
import ssdMobileNetV2Service from "./ssdMobileNetV2Service";
|
||||
import yoloFaceDetectionService from "./yoloFaceDetectionService";
|
||||
|
||||
export class MLFactory {
|
||||
|
|
@ -49,26 +43,6 @@ export class MLFactory {
|
|||
throw Error("Unknon face detection method: " + method);
|
||||
}
|
||||
|
||||
public static getObjectDetectionService(
|
||||
method: ObjectDetectionMethod,
|
||||
): ObjectDetectionService {
|
||||
if (method === "SSDMobileNetV2") {
|
||||
return ssdMobileNetV2Service;
|
||||
}
|
||||
|
||||
throw Error("Unknown object detection method: " + method);
|
||||
}
|
||||
|
||||
public static getSceneDetectionService(
|
||||
method: SceneDetectionMethod,
|
||||
): SceneDetectionService {
|
||||
if (method === "ImageScene") {
|
||||
return imageSceneService;
|
||||
}
|
||||
|
||||
throw Error("Unknown scene detection method: " + method);
|
||||
}
|
||||
|
||||
public static getFaceCropService(method: FaceCropMethod) {
|
||||
if (method === "ArcFace") {
|
||||
return arcfaceCropService;
|
||||
|
|
@ -147,15 +121,12 @@ export class LocalMLSyncContext implements MLSyncContext {
|
|||
public blurDetectionService: BlurDetectionService;
|
||||
public faceEmbeddingService: FaceEmbeddingService;
|
||||
public faceClusteringService: ClusteringService;
|
||||
public objectDetectionService: ObjectDetectionService;
|
||||
public sceneDetectionService: SceneDetectionService;
|
||||
|
||||
public localFilesMap: Map<number, EnteFile>;
|
||||
public outOfSyncFiles: EnteFile[];
|
||||
public nSyncedFiles: number;
|
||||
public nSyncedFaces: number;
|
||||
public allSyncedFacesMap?: Map<number, Array<Face>>;
|
||||
public tsne?: any;
|
||||
|
||||
public error?: Error;
|
||||
|
||||
|
|
@ -202,13 +173,6 @@ export class LocalMLSyncContext implements MLSyncContext {
|
|||
this.config.faceClustering.method,
|
||||
);
|
||||
|
||||
this.objectDetectionService = MLFactory.getObjectDetectionService(
|
||||
this.config.objectDetection.method,
|
||||
);
|
||||
this.sceneDetectionService = MLFactory.getSceneDetectionService(
|
||||
this.config.sceneDetection.method,
|
||||
);
|
||||
|
||||
this.outOfSyncFiles = [];
|
||||
this.nSyncedFiles = 0;
|
||||
this.nSyncedFaces = 0;
|
||||
|
|
|
|||
|
|
@ -2,9 +2,6 @@ import log from "@/next/log";
|
|||
import { APPS } from "@ente/shared/apps/constants";
|
||||
import ComlinkCryptoWorker from "@ente/shared/crypto";
|
||||
import { CustomError, parseUploadErrorCodes } from "@ente/shared/error";
|
||||
import "@tensorflow/tfjs-backend-cpu";
|
||||
import "@tensorflow/tfjs-backend-webgl";
|
||||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import { MAX_ML_SYNC_ERROR_COUNT } from "constants/mlConfig";
|
||||
import downloadManager from "services/download";
|
||||
import { putEmbedding } from "services/embeddingService";
|
||||
|
|
@ -21,13 +18,10 @@ import { LocalFileMlDataToServerFileMl } from "utils/machineLearning/mldataMappe
|
|||
import mlIDbStorage from "utils/storage/mlIDbStorage";
|
||||
import FaceService from "./faceService";
|
||||
import { MLFactory } from "./machineLearningFactory";
|
||||
import ObjectService from "./objectService";
|
||||
import PeopleService from "./peopleService";
|
||||
import ReaderService from "./readerService";
|
||||
|
||||
class MachineLearningService {
|
||||
private initialized = false;
|
||||
|
||||
private localSyncContext: Promise<MLSyncContext>;
|
||||
private syncContext: Promise<MLSyncContext>;
|
||||
|
||||
|
|
@ -58,12 +52,6 @@ class MachineLearningService {
|
|||
await this.syncIndex(syncContext);
|
||||
}
|
||||
|
||||
// tf.engine().endScope();
|
||||
|
||||
// if (syncContext.config.tsne) {
|
||||
// await this.runTSNE(syncContext);
|
||||
// }
|
||||
|
||||
const mlSyncResult: MLSyncResult = {
|
||||
nOutOfSyncFiles: syncContext.outOfSyncFiles.length,
|
||||
nSyncedFiles: syncContext.nSyncedFiles,
|
||||
|
|
@ -73,14 +61,10 @@ class MachineLearningService {
|
|||
.length,
|
||||
nFaceNoise:
|
||||
syncContext.mlLibraryData?.faceClusteringResults?.noise.length,
|
||||
tsne: syncContext.tsne,
|
||||
error: syncContext.error,
|
||||
};
|
||||
// log.info('[MLService] sync results: ', mlSyncResult);
|
||||
|
||||
// await syncContext.dispose();
|
||||
log.info("Final TF Memory stats: ", JSON.stringify(tf.memory()));
|
||||
|
||||
return mlSyncResult;
|
||||
}
|
||||
|
||||
|
|
@ -183,50 +167,6 @@ class MachineLearningService {
|
|||
log.info("getOutOfSyncFiles", Date.now() - startTime, "ms");
|
||||
}
|
||||
|
||||
// TODO: optimize, use indexdb indexes, move facecrops to cache to reduce io
|
||||
// remove, already done
|
||||
private async getUniqueOutOfSyncFilesNoIdx(
|
||||
syncContext: MLSyncContext,
|
||||
files: EnteFile[],
|
||||
) {
|
||||
const limit = syncContext.config.batchSize;
|
||||
const mlVersion = syncContext.config.mlVersion;
|
||||
const uniqueFiles: Map<number, EnteFile> = new Map<number, EnteFile>();
|
||||
for (let i = 0; uniqueFiles.size < limit && i < files.length; i++) {
|
||||
const mlFileData = await this.getMLFileData(files[i].id);
|
||||
const mlFileVersion = mlFileData?.mlVersion || 0;
|
||||
if (
|
||||
!uniqueFiles.has(files[i].id) &&
|
||||
(!mlFileData?.errorCount || mlFileData.errorCount < 2) &&
|
||||
(mlFileVersion < mlVersion ||
|
||||
syncContext.config.imageSource !== mlFileData.imageSource)
|
||||
) {
|
||||
uniqueFiles.set(files[i].id, files[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return [...uniqueFiles.values()];
|
||||
}
|
||||
|
||||
private async getOutOfSyncFilesNoIdx(syncContext: MLSyncContext) {
|
||||
const existingFilesMap = await this.getLocalFilesMap(syncContext);
|
||||
// existingFiles.sort(
|
||||
// (a, b) => b.metadata.creationTime - a.metadata.creationTime
|
||||
// );
|
||||
console.time("getUniqueOutOfSyncFiles");
|
||||
syncContext.outOfSyncFiles = await this.getUniqueOutOfSyncFilesNoIdx(
|
||||
syncContext,
|
||||
[...existingFilesMap.values()],
|
||||
);
|
||||
log.info("getUniqueOutOfSyncFiles");
|
||||
log.info(
|
||||
"Got unique outOfSyncFiles: ",
|
||||
syncContext.outOfSyncFiles.length,
|
||||
"for batchSize: ",
|
||||
syncContext.config.batchSize,
|
||||
);
|
||||
}
|
||||
|
||||
private async syncFiles(syncContext: MLSyncContext) {
|
||||
try {
|
||||
const functions = syncContext.outOfSyncFiles.map(
|
||||
|
|
@ -295,7 +235,6 @@ class MachineLearningService {
|
|||
userID: number,
|
||||
enteFile: EnteFile,
|
||||
localFile?: globalThis.File,
|
||||
textDetectionTimeoutIndex?: number,
|
||||
): Promise<MlFileData | Error> {
|
||||
const syncContext = await this.getLocalSyncContext(token, userID);
|
||||
|
||||
|
|
@ -304,7 +243,6 @@ class MachineLearningService {
|
|||
syncContext,
|
||||
enteFile,
|
||||
localFile,
|
||||
textDetectionTimeoutIndex,
|
||||
);
|
||||
|
||||
if (syncContext.nSyncedFiles >= syncContext.config.batchSize) {
|
||||
|
|
@ -322,19 +260,15 @@ class MachineLearningService {
|
|||
syncContext: MLSyncContext,
|
||||
enteFile: EnteFile,
|
||||
localFile?: globalThis.File,
|
||||
textDetectionTimeoutIndex?: number,
|
||||
): Promise<MlFileData> {
|
||||
try {
|
||||
console.log(
|
||||
"Start index for ",
|
||||
enteFile.title ?? "no title",
|
||||
enteFile.id,
|
||||
`Indexing ${enteFile.title ?? "<untitled>"} ${enteFile.id}`,
|
||||
);
|
||||
const mlFileData = await this.syncFile(
|
||||
syncContext,
|
||||
enteFile,
|
||||
localFile,
|
||||
textDetectionTimeoutIndex,
|
||||
);
|
||||
syncContext.nSyncedFaces += mlFileData.faces?.length || 0;
|
||||
syncContext.nSyncedFiles += 1;
|
||||
|
|
@ -363,16 +297,8 @@ class MachineLearningService {
|
|||
throw error;
|
||||
}
|
||||
|
||||
await this.persistMLFileSyncError(syncContext, enteFile, error);
|
||||
await this.persistMLFileSyncError(enteFile, error);
|
||||
syncContext.nSyncedFiles += 1;
|
||||
} finally {
|
||||
console.log(
|
||||
"done index for ",
|
||||
enteFile.title ?? "no title",
|
||||
enteFile.id,
|
||||
);
|
||||
// addLogLine('TF Memory stats: ', JSON.stringify(tf.memory()));
|
||||
log.info("TF Memory stats: ", JSON.stringify(tf.memory()));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -380,8 +306,6 @@ class MachineLearningService {
|
|||
syncContext: MLSyncContext,
|
||||
enteFile: EnteFile,
|
||||
localFile?: globalThis.File,
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
textDetectionTimeoutIndex?: number,
|
||||
) {
|
||||
console.log("Syncing for file" + enteFile.title);
|
||||
const fileContext: MLSyncFileContext = { enteFile, localFile };
|
||||
|
|
@ -406,34 +330,17 @@ class MachineLearningService {
|
|||
await ReaderService.getImageBitmap(syncContext, fileContext);
|
||||
await Promise.all([
|
||||
this.syncFileAnalyzeFaces(syncContext, fileContext),
|
||||
// ObjectService.syncFileObjectDetections(
|
||||
// syncContext,
|
||||
// fileContext
|
||||
// ),
|
||||
// TextService.syncFileTextDetections(
|
||||
// syncContext,
|
||||
// fileContext,
|
||||
// textDetectionTimeoutIndex
|
||||
// ),
|
||||
]);
|
||||
newMlFile.errorCount = 0;
|
||||
newMlFile.lastErrorMessage = undefined;
|
||||
await this.persistOnServer(newMlFile, enteFile);
|
||||
await this.persistMLFileData(syncContext, newMlFile);
|
||||
await mlIDbStorage.putFile(newMlFile);
|
||||
} catch (e) {
|
||||
log.error("ml detection failed", e);
|
||||
newMlFile.mlVersion = oldMlFile.mlVersion;
|
||||
throw e;
|
||||
} finally {
|
||||
fileContext.tfImage && fileContext.tfImage.dispose();
|
||||
fileContext.imageBitmap && fileContext.imageBitmap.close();
|
||||
// log.info('8 TF Memory stats: ',JSON.stringify(tf.memory()));
|
||||
|
||||
// TODO: enable once faceId changes go in
|
||||
// await removeOldFaceCrops(
|
||||
// fileContext.oldMlFile,
|
||||
// fileContext.newMlFile
|
||||
// );
|
||||
}
|
||||
|
||||
return newMlFile;
|
||||
|
|
@ -458,38 +365,11 @@ class MachineLearningService {
|
|||
log.info("putEmbedding response: ", res);
|
||||
}
|
||||
|
||||
public async init() {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
await tf.ready();
|
||||
|
||||
log.info("01 TF Memory stats: ", JSON.stringify(tf.memory()));
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
public async dispose() {
|
||||
this.initialized = false;
|
||||
}
|
||||
|
||||
private async getMLFileData(fileId: number) {
|
||||
return mlIDbStorage.getFile(fileId);
|
||||
}
|
||||
|
||||
private async persistMLFileData(
|
||||
syncContext: MLSyncContext,
|
||||
mlFileData: MlFileData,
|
||||
) {
|
||||
mlIDbStorage.putFile(mlFileData);
|
||||
}
|
||||
|
||||
private async persistMLFileSyncError(
|
||||
syncContext: MLSyncContext,
|
||||
enteFile: EnteFile,
|
||||
e: Error,
|
||||
) {
|
||||
private async persistMLFileSyncError(enteFile: EnteFile, e: Error) {
|
||||
try {
|
||||
await mlIDbStorage.upsertFileInTx(enteFile.id, (mlFileData) => {
|
||||
if (!mlFileData) {
|
||||
|
|
@ -522,8 +402,6 @@ class MachineLearningService {
|
|||
|
||||
await PeopleService.syncPeopleIndex(syncContext);
|
||||
|
||||
await ObjectService.syncThingsIndex(syncContext);
|
||||
|
||||
await this.persistMLLibraryData(syncContext);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
import { JobResult } from "types/common/job";
|
||||
import { MLSyncResult } from "types/machineLearning";
|
||||
import { SimpleJob } from "utils/common/job";
|
||||
|
||||
export interface MLSyncJobResult extends JobResult {
|
||||
mlSyncResult: MLSyncResult;
|
||||
}
|
||||
|
||||
export class MLSyncJob extends SimpleJob<MLSyncJobResult> {}
|
||||
|
|
@ -5,18 +5,26 @@ import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers";
|
|||
import { FILE_TYPE } from "constants/file";
|
||||
import debounce from "debounce";
|
||||
import PQueue from "p-queue";
|
||||
import { JobResult } from "types/common/job";
|
||||
import { EnteFile } from "types/file";
|
||||
import { MLSyncResult } from "types/machineLearning";
|
||||
import { getDedicatedMLWorker } from "utils/comlink/ComlinkMLWorker";
|
||||
import { SimpleJob } from "utils/common/job";
|
||||
import { logQueueStats } from "utils/machineLearning";
|
||||
import { getMLSyncJobConfig } from "utils/machineLearning/config";
|
||||
import mlIDbStorage from "utils/storage/mlIDbStorage";
|
||||
import { DedicatedMLWorker } from "worker/ml.worker";
|
||||
import { MLSyncJob, MLSyncJobResult } from "./mlSyncJob";
|
||||
|
||||
const LIVE_SYNC_IDLE_DEBOUNCE_SEC = 30;
|
||||
const LIVE_SYNC_QUEUE_TIMEOUT_SEC = 300;
|
||||
const LOCAL_FILES_UPDATED_DEBOUNCE_SEC = 30;
|
||||
|
||||
export interface MLSyncJobResult extends JobResult {
|
||||
mlSyncResult: MLSyncResult;
|
||||
}
|
||||
|
||||
export class MLSyncJob extends SimpleJob<MLSyncJobResult> {}
|
||||
|
||||
class MLWorkManager {
|
||||
private mlSyncJob: MLSyncJob;
|
||||
private syncJobWorker: ComlinkWorker<typeof DedicatedMLWorker>;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,7 @@
|
|||
import log from "@/next/log";
|
||||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import {
|
||||
MOBILEFACENET_EMBEDDING_SIZE,
|
||||
MOBILEFACENET_FACE_SIZE,
|
||||
} from "constants/mlConfig";
|
||||
// import { TFLiteModel } from "@tensorflow/tfjs-tflite";
|
||||
// import PQueue from "p-queue";
|
||||
import {
|
||||
FaceEmbedding,
|
||||
FaceEmbeddingMethod,
|
||||
|
|
@ -18,12 +14,6 @@ import {
|
|||
// import { env } from "onnxruntime-web";
|
||||
const ort: any = {};
|
||||
|
||||
import {
|
||||
clamp,
|
||||
getPixelBilinear,
|
||||
normalizePixelBetweenMinus1And1,
|
||||
} from "utils/image";
|
||||
|
||||
// TODO(MR): onnx-yolo
|
||||
// env.wasm.wasmPaths = "/js/onnx/";
|
||||
class MobileFaceNetEmbeddingService implements FaceEmbeddingService {
|
||||
|
|
@ -33,12 +23,12 @@ class MobileFaceNetEmbeddingService implements FaceEmbeddingService {
|
|||
public method: Versioned<FaceEmbeddingMethod>;
|
||||
public faceSize: number;
|
||||
|
||||
public constructor(faceSize: number = MOBILEFACENET_FACE_SIZE) {
|
||||
public constructor() {
|
||||
this.method = {
|
||||
value: "MobileFaceNet",
|
||||
version: 2,
|
||||
};
|
||||
this.faceSize = faceSize;
|
||||
this.faceSize = MOBILEFACENET_FACE_SIZE;
|
||||
// TODO: set timeout
|
||||
}
|
||||
|
||||
|
|
@ -73,85 +63,6 @@ class MobileFaceNetEmbeddingService implements FaceEmbeddingService {
|
|||
return this.onnxInferenceSession;
|
||||
}
|
||||
|
||||
private preprocessImageBitmapToFloat32(
|
||||
imageBitmap: ImageBitmap,
|
||||
requiredWidth: number = this.faceSize,
|
||||
requiredHeight: number = this.faceSize,
|
||||
maintainAspectRatio: boolean = true,
|
||||
normFunction: (
|
||||
pixelValue: number,
|
||||
) => number = normalizePixelBetweenMinus1And1,
|
||||
) {
|
||||
// Create an OffscreenCanvas and set its size
|
||||
const offscreenCanvas = new OffscreenCanvas(
|
||||
imageBitmap.width,
|
||||
imageBitmap.height,
|
||||
);
|
||||
const ctx = offscreenCanvas.getContext("2d");
|
||||
ctx.drawImage(imageBitmap, 0, 0, imageBitmap.width, imageBitmap.height);
|
||||
const imageData = ctx.getImageData(
|
||||
0,
|
||||
0,
|
||||
imageBitmap.width,
|
||||
imageBitmap.height,
|
||||
);
|
||||
const pixelData = imageData.data;
|
||||
|
||||
let scaleW = requiredWidth / imageBitmap.width;
|
||||
let scaleH = requiredHeight / imageBitmap.height;
|
||||
if (maintainAspectRatio) {
|
||||
const scale = Math.min(
|
||||
requiredWidth / imageBitmap.width,
|
||||
requiredHeight / imageBitmap.height,
|
||||
);
|
||||
scaleW = scale;
|
||||
scaleH = scale;
|
||||
}
|
||||
const scaledWidth = clamp(
|
||||
Math.round(imageBitmap.width * scaleW),
|
||||
0,
|
||||
requiredWidth,
|
||||
);
|
||||
const scaledHeight = clamp(
|
||||
Math.round(imageBitmap.height * scaleH),
|
||||
0,
|
||||
requiredHeight,
|
||||
);
|
||||
|
||||
const processedImage = new Float32Array(
|
||||
1 * requiredWidth * requiredHeight * 3,
|
||||
);
|
||||
log.info("loaded mobileFaceNetModel: ", tf.getBackend());
|
||||
|
||||
// Populate the Float32Array with normalized pixel values
|
||||
for (let h = 0; h < requiredHeight; h++) {
|
||||
for (let w = 0; w < requiredWidth; w++) {
|
||||
let pixel: {
|
||||
r: number;
|
||||
g: number;
|
||||
b: number;
|
||||
};
|
||||
if (w >= scaledWidth || h >= scaledHeight) {
|
||||
pixel = { r: 114, g: 114, b: 114 };
|
||||
} else {
|
||||
pixel = getPixelBilinear(
|
||||
w / scaleW,
|
||||
h / scaleH,
|
||||
pixelData,
|
||||
imageBitmap.width,
|
||||
imageBitmap.height,
|
||||
);
|
||||
}
|
||||
const pixelIndex = 3 * (h * requiredWidth + w);
|
||||
processedImage[pixelIndex] = normFunction(pixel.r);
|
||||
processedImage[pixelIndex + 1] = normFunction(pixel.g);
|
||||
processedImage[pixelIndex + 2] = normFunction(pixel.b);
|
||||
}
|
||||
}
|
||||
|
||||
return processedImage;
|
||||
}
|
||||
|
||||
public async getFaceEmbeddings(
|
||||
faceData: Float32Array,
|
||||
): Promise<Array<FaceEmbedding>> {
|
||||
|
|
|
|||
|
|
@ -1,146 +0,0 @@
|
|||
import log from "@/next/log";
|
||||
import {
|
||||
DetectedObject,
|
||||
MLSyncContext,
|
||||
MLSyncFileContext,
|
||||
Thing,
|
||||
} from "types/machineLearning";
|
||||
import {
|
||||
getAllObjectsFromMap,
|
||||
getObjectId,
|
||||
isDifferentOrOld,
|
||||
} from "utils/machineLearning";
|
||||
import mlIDbStorage from "utils/storage/mlIDbStorage";
|
||||
import ReaderService from "./readerService";
|
||||
|
||||
class ObjectService {
|
||||
async syncFileObjectDetections(
|
||||
syncContext: MLSyncContext,
|
||||
fileContext: MLSyncFileContext,
|
||||
) {
|
||||
const startTime = Date.now();
|
||||
const { oldMlFile, newMlFile } = fileContext;
|
||||
if (
|
||||
!isDifferentOrOld(
|
||||
oldMlFile?.objectDetectionMethod,
|
||||
syncContext.objectDetectionService.method,
|
||||
) &&
|
||||
!isDifferentOrOld(
|
||||
oldMlFile?.sceneDetectionMethod,
|
||||
syncContext.sceneDetectionService.method,
|
||||
) &&
|
||||
oldMlFile?.imageSource === syncContext.config.imageSource
|
||||
) {
|
||||
newMlFile.objects = oldMlFile?.objects;
|
||||
newMlFile.imageSource = oldMlFile.imageSource;
|
||||
newMlFile.imageDimensions = oldMlFile.imageDimensions;
|
||||
newMlFile.objectDetectionMethod = oldMlFile.objectDetectionMethod;
|
||||
newMlFile.sceneDetectionMethod = oldMlFile.sceneDetectionMethod;
|
||||
return;
|
||||
}
|
||||
|
||||
newMlFile.objectDetectionMethod =
|
||||
syncContext.objectDetectionService.method;
|
||||
newMlFile.sceneDetectionMethod =
|
||||
syncContext.sceneDetectionService.method;
|
||||
|
||||
fileContext.newDetection = true;
|
||||
const imageBitmap = await ReaderService.getImageBitmap(
|
||||
syncContext,
|
||||
fileContext,
|
||||
);
|
||||
const objectDetections =
|
||||
await syncContext.objectDetectionService.detectObjects(
|
||||
imageBitmap,
|
||||
syncContext.config.objectDetection.maxNumBoxes,
|
||||
syncContext.config.objectDetection.minScore,
|
||||
);
|
||||
objectDetections.push(
|
||||
...(await syncContext.sceneDetectionService.detectScenes(
|
||||
imageBitmap,
|
||||
syncContext.config.sceneDetection.minScore,
|
||||
)),
|
||||
);
|
||||
// log.info('3 TF Memory stats: ',JSON.stringify(tf.memory()));
|
||||
// TODO: reenable faces filtering based on width
|
||||
const detectedObjects = objectDetections?.map((detection) => {
|
||||
return {
|
||||
fileID: fileContext.enteFile.id,
|
||||
detection,
|
||||
} as DetectedObject;
|
||||
});
|
||||
newMlFile.objects = detectedObjects?.map((detectedObject) => ({
|
||||
...detectedObject,
|
||||
id: getObjectId(detectedObject, newMlFile.imageDimensions),
|
||||
className: detectedObject.detection.class,
|
||||
}));
|
||||
// ?.filter((f) =>
|
||||
// f.box.width > syncContext.config.faceDetection.minFaceSize
|
||||
// );
|
||||
log.info(
|
||||
`object detection time taken ${fileContext.enteFile.id}`,
|
||||
Date.now() - startTime,
|
||||
"ms",
|
||||
);
|
||||
|
||||
log.info("[MLService] Detected Objects: ", newMlFile.objects?.length);
|
||||
}
|
||||
|
||||
async getAllSyncedObjectsMap(syncContext: MLSyncContext) {
|
||||
if (syncContext.allSyncedObjectsMap) {
|
||||
return syncContext.allSyncedObjectsMap;
|
||||
}
|
||||
|
||||
syncContext.allSyncedObjectsMap = await mlIDbStorage.getAllObjectsMap();
|
||||
return syncContext.allSyncedObjectsMap;
|
||||
}
|
||||
|
||||
public async clusterThings(syncContext: MLSyncContext): Promise<Thing[]> {
|
||||
const allObjectsMap = await this.getAllSyncedObjectsMap(syncContext);
|
||||
const allObjects = getAllObjectsFromMap(allObjectsMap);
|
||||
const objectClusters = new Map<string, number[]>();
|
||||
allObjects.map((object) => {
|
||||
if (!objectClusters.has(object.detection.class)) {
|
||||
objectClusters.set(object.detection.class, []);
|
||||
}
|
||||
const objectsInCluster = objectClusters.get(object.detection.class);
|
||||
objectsInCluster.push(object.fileID);
|
||||
});
|
||||
return [...objectClusters.entries()].map(([className, files], id) => ({
|
||||
id,
|
||||
name: className,
|
||||
files,
|
||||
}));
|
||||
}
|
||||
|
||||
async syncThingsIndex(syncContext: MLSyncContext) {
|
||||
const filesVersion = await mlIDbStorage.getIndexVersion("files");
|
||||
log.info("things", await mlIDbStorage.getIndexVersion("things"));
|
||||
if (filesVersion <= (await mlIDbStorage.getIndexVersion("things"))) {
|
||||
log.info(
|
||||
"[MLService] Skipping people index as already synced to latest version",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const things = await this.clusterThings(syncContext);
|
||||
|
||||
if (!things || things.length < 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
await mlIDbStorage.clearAllThings();
|
||||
|
||||
for (const thing of things) {
|
||||
await mlIDbStorage.putThing(thing);
|
||||
}
|
||||
|
||||
await mlIDbStorage.setIndexVersion("things", filesVersion);
|
||||
}
|
||||
|
||||
async getAllThings() {
|
||||
return await mlIDbStorage.getAllThings();
|
||||
}
|
||||
}
|
||||
|
||||
export default new ObjectService();
|
||||
|
|
@ -16,7 +16,6 @@ class ReaderService {
|
|||
if (fileContext.imageBitmap) {
|
||||
return fileContext.imageBitmap;
|
||||
}
|
||||
// log.info('1 TF Memory stats: ',JSON.stringify(tf.memory()));
|
||||
if (fileContext.localFile) {
|
||||
if (
|
||||
fileContext.enteFile.metadata.fileType !== FILE_TYPE.IMAGE
|
||||
|
|
@ -47,7 +46,6 @@ class ReaderService {
|
|||
fileContext.newMlFile.imageSource = syncContext.config.imageSource;
|
||||
const { width, height } = fileContext.imageBitmap;
|
||||
fileContext.newMlFile.imageDimensions = { width, height };
|
||||
// log.info('2 TF Memory stats: ',JSON.stringify(tf.memory()));
|
||||
|
||||
return fileContext.imageBitmap;
|
||||
} catch (e) {
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
import log from "@/next/log";
|
||||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import {
|
||||
ObjectDetection,
|
||||
ObjectDetectionMethod,
|
||||
ObjectDetectionService,
|
||||
Versioned,
|
||||
} from "types/machineLearning";
|
||||
|
||||
import * as SSDMobileNet from "@tensorflow-models/coco-ssd";
|
||||
import { OBJECT_DETECTION_IMAGE_SIZE } from "constants/mlConfig";
|
||||
import { resizeToSquare } from "utils/image";
|
||||
|
||||
class SSDMobileNetV2 implements ObjectDetectionService {
|
||||
private ssdMobileNetV2Model: SSDMobileNet.ObjectDetection;
|
||||
public method: Versioned<ObjectDetectionMethod>;
|
||||
private ready: Promise<void>;
|
||||
|
||||
public constructor() {
|
||||
this.method = {
|
||||
value: "SSDMobileNetV2",
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
private async init() {
|
||||
this.ssdMobileNetV2Model = await SSDMobileNet.load({
|
||||
base: "mobilenet_v2",
|
||||
modelUrl: "/models/ssdmobilenet/model.json",
|
||||
});
|
||||
log.info("loaded ssdMobileNetV2Model", tf.getBackend());
|
||||
}
|
||||
|
||||
private async getSSDMobileNetV2Model() {
|
||||
if (!this.ready) {
|
||||
this.ready = this.init();
|
||||
}
|
||||
await this.ready;
|
||||
return this.ssdMobileNetV2Model;
|
||||
}
|
||||
|
||||
public async detectObjects(
|
||||
image: ImageBitmap,
|
||||
maxNumberBoxes: number,
|
||||
minScore: number,
|
||||
): Promise<ObjectDetection[]> {
|
||||
const ssdMobileNetV2Model = await this.getSSDMobileNetV2Model();
|
||||
const resized = resizeToSquare(image, OBJECT_DETECTION_IMAGE_SIZE);
|
||||
const tfImage = tf.browser.fromPixels(resized.image);
|
||||
const detections = await ssdMobileNetV2Model.detect(
|
||||
tfImage,
|
||||
maxNumberBoxes,
|
||||
minScore,
|
||||
);
|
||||
tfImage.dispose();
|
||||
return detections;
|
||||
}
|
||||
|
||||
public async dispose() {
|
||||
const ssdMobileNetV2Model = await this.getSSDMobileNetV2Model();
|
||||
ssdMobileNetV2Model?.dispose();
|
||||
this.ssdMobileNetV2Model = null;
|
||||
}
|
||||
}
|
||||
|
||||
export default new SSDMobileNetV2();
|
||||
|
|
@ -1,7 +1,11 @@
|
|||
import { MAX_FACE_DISTANCE_PERCENT } from "constants/mlConfig";
|
||||
import {
|
||||
BLAZEFACE_FACE_SIZE,
|
||||
MAX_FACE_DISTANCE_PERCENT,
|
||||
} from "constants/mlConfig";
|
||||
Matrix,
|
||||
applyToPoint,
|
||||
compose,
|
||||
scale,
|
||||
translate,
|
||||
} from "transformation-matrix";
|
||||
import { Dimensions } from "types/image";
|
||||
import {
|
||||
FaceDetection,
|
||||
|
|
@ -15,12 +19,6 @@ import {
|
|||
normalizePixelBetween0And1,
|
||||
} from "utils/image";
|
||||
import { newBox } from "utils/machineLearning";
|
||||
import { removeDuplicateDetections } from "utils/machineLearning/faceDetection";
|
||||
import {
|
||||
computeTransformToBox,
|
||||
transformBox,
|
||||
transformPoints,
|
||||
} from "utils/machineLearning/transform";
|
||||
import { Box, Point } from "../../../thirdparty/face-api/classes";
|
||||
|
||||
// TODO(MR): onnx-yolo
|
||||
|
|
@ -35,14 +33,12 @@ class YoloFaceDetectionService implements FaceDetectionService {
|
|||
// private onnxInferenceSession?: ort.InferenceSession;
|
||||
private onnxInferenceSession?: any;
|
||||
public method: Versioned<FaceDetectionMethod>;
|
||||
private desiredFaceSize;
|
||||
|
||||
public constructor(desiredFaceSize: number = BLAZEFACE_FACE_SIZE) {
|
||||
public constructor() {
|
||||
this.method = {
|
||||
value: "YoloFace",
|
||||
version: 1,
|
||||
};
|
||||
this.desiredFaceSize = desiredFaceSize;
|
||||
}
|
||||
|
||||
private async initOnnx() {
|
||||
|
|
@ -329,3 +325,97 @@ class YoloFaceDetectionService implements FaceDetectionService {
|
|||
}
|
||||
|
||||
export default new YoloFaceDetectionService();
|
||||
|
||||
import { euclidean } from "hdbscan";
|
||||
|
||||
/**
|
||||
* Removes duplicate face detections from an array of detections.
|
||||
*
|
||||
* This function sorts the detections by their probability in descending order, then iterates over them.
|
||||
* For each detection, it calculates the Euclidean distance to all other detections.
|
||||
* If the distance is less than or equal to the specified threshold (`withinDistance`), the other detection is considered a duplicate and is removed.
|
||||
*
|
||||
* @param detections - An array of face detections to remove duplicates from.
|
||||
* @param withinDistance - The maximum Euclidean distance between two detections for them to be considered duplicates.
|
||||
*
|
||||
* @returns An array of face detections with duplicates removed.
|
||||
*/
|
||||
function removeDuplicateDetections(
|
||||
detections: Array<FaceDetection>,
|
||||
withinDistance: number,
|
||||
) {
|
||||
// console.time('removeDuplicates');
|
||||
detections.sort((a, b) => b.probability - a.probability);
|
||||
const isSelected = new Map<number, boolean>();
|
||||
for (let i = 0; i < detections.length; i++) {
|
||||
if (isSelected.get(i) === false) {
|
||||
continue;
|
||||
}
|
||||
isSelected.set(i, true);
|
||||
for (let j = i + 1; j < detections.length; j++) {
|
||||
if (isSelected.get(j) === false) {
|
||||
continue;
|
||||
}
|
||||
const centeri = getDetectionCenter(detections[i]);
|
||||
const centerj = getDetectionCenter(detections[j]);
|
||||
const dist = euclidean(
|
||||
[centeri.x, centeri.y],
|
||||
[centerj.x, centerj.y],
|
||||
);
|
||||
if (dist <= withinDistance) {
|
||||
isSelected.set(j, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const uniques: Array<FaceDetection> = [];
|
||||
for (let i = 0; i < detections.length; i++) {
|
||||
isSelected.get(i) && uniques.push(detections[i]);
|
||||
}
|
||||
// console.timeEnd('removeDuplicates');
|
||||
return uniques;
|
||||
}
|
||||
|
||||
function getDetectionCenter(detection: FaceDetection) {
|
||||
const center = new Point(0, 0);
|
||||
// TODO: first 4 landmarks is applicable to blazeface only
|
||||
// this needs to consider eyes, nose and mouth landmarks to take center
|
||||
detection.landmarks?.slice(0, 4).forEach((p) => {
|
||||
center.x += p.x;
|
||||
center.y += p.y;
|
||||
});
|
||||
|
||||
return center.div({ x: 4, y: 4 });
|
||||
}
|
||||
|
||||
function computeTransformToBox(inBox: Box, toBox: Box): Matrix {
|
||||
return compose(
|
||||
translate(toBox.x, toBox.y),
|
||||
scale(toBox.width / inBox.width, toBox.height / inBox.height),
|
||||
);
|
||||
}
|
||||
|
||||
function transformPoint(point: Point, transform: Matrix) {
|
||||
const txdPoint = applyToPoint(transform, point);
|
||||
return new Point(txdPoint.x, txdPoint.y);
|
||||
}
|
||||
|
||||
function transformPoints(points: Point[], transform: Matrix) {
|
||||
return points?.map((p) => transformPoint(p, transform));
|
||||
}
|
||||
|
||||
function transformBox(box: Box, transform: Matrix) {
|
||||
const topLeft = transformPoint(box.topLeft, transform);
|
||||
const bottomRight = transformPoint(box.bottomRight, transform);
|
||||
|
||||
return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y);
|
||||
}
|
||||
|
||||
function newBoxFromPoints(
|
||||
left: number,
|
||||
top: number,
|
||||
right: number,
|
||||
bottom: number,
|
||||
) {
|
||||
return new Box({ left, top, right, bottom });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { t } from "i18next";
|
|||
import { Collection } from "types/collection";
|
||||
import { EntityType, LocationTag, LocationTagData } from "types/entity";
|
||||
import { EnteFile } from "types/file";
|
||||
import { Person, Thing } from "types/machineLearning";
|
||||
import { Person } from "types/machineLearning";
|
||||
import {
|
||||
ClipSearchScores,
|
||||
DateValue,
|
||||
|
|
@ -25,7 +25,6 @@ import { clipService, computeClipMatchScore } from "./clip-service";
|
|||
import { getLocalEmbeddings } from "./embeddingService";
|
||||
import { getLatestEntities } from "./entityService";
|
||||
import locationSearchService, { City } from "./locationSearchService";
|
||||
import ObjectService from "./machineLearning/objectService";
|
||||
|
||||
const DIGITS = new Set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]);
|
||||
|
||||
|
|
@ -56,7 +55,6 @@ export const getAutoCompleteSuggestions =
|
|||
getFileNameSuggestion(searchPhrase, files),
|
||||
getFileCaptionSuggestion(searchPhrase, files),
|
||||
...(await getLocationSuggestions(searchPhrase)),
|
||||
...(await getThingSuggestion(searchPhrase)),
|
||||
].filter((suggestion) => !!suggestion);
|
||||
|
||||
return convertSuggestionsToOptions(suggestions);
|
||||
|
|
@ -289,19 +287,6 @@ async function getLocationSuggestions(searchPhrase: string) {
|
|||
return [...locationTagSuggestions, ...citySearchSuggestions];
|
||||
}
|
||||
|
||||
async function getThingSuggestion(searchPhrase: string): Promise<Suggestion[]> {
|
||||
const thingResults = await searchThing(searchPhrase);
|
||||
|
||||
return thingResults.map(
|
||||
(searchResult) =>
|
||||
({
|
||||
type: SuggestionType.THING,
|
||||
value: searchResult,
|
||||
label: searchResult.name,
|
||||
}) as Suggestion,
|
||||
);
|
||||
}
|
||||
|
||||
async function getClipSuggestion(searchPhrase: string): Promise<Suggestion> {
|
||||
try {
|
||||
if (!clipService.isPlatformSupported()) {
|
||||
|
|
@ -389,13 +374,6 @@ async function searchLocationTag(searchPhrase: string): Promise<LocationTag[]> {
|
|||
return matchedLocationTags;
|
||||
}
|
||||
|
||||
async function searchThing(searchPhrase: string) {
|
||||
const things = await ObjectService.getAllThings();
|
||||
return things.filter((thing) =>
|
||||
thing.name.toLocaleLowerCase().includes(searchPhrase),
|
||||
);
|
||||
}
|
||||
|
||||
async function searchClip(searchPhrase: string): Promise<ClipSearchScores> {
|
||||
const imageEmbeddings = await getLocalEmbeddings();
|
||||
const textEmbedding = await clipService.getTextEmbedding(searchPhrase);
|
||||
|
|
@ -445,10 +423,9 @@ function convertSuggestionToSearchQuery(option: Suggestion): Search {
|
|||
case SuggestionType.PERSON:
|
||||
return { person: option.value as Person };
|
||||
|
||||
case SuggestionType.THING:
|
||||
return { thing: option.value as Thing };
|
||||
case SuggestionType.FILE_TYPE:
|
||||
return { fileType: option.value as FILE_TYPE };
|
||||
|
||||
case SuggestionType.CLIP:
|
||||
return { clip: option.value as ClipSearchScores };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
export const ARCFACE_LANDMARKS = [
|
||||
[38.2946, 51.6963],
|
||||
[73.5318, 51.5014],
|
||||
[56.0252, 71.7366],
|
||||
[56.1396, 92.2848],
|
||||
] as Array<[number, number]>;
|
||||
|
||||
export const ARCFACE_LANDMARKS_FACE_SIZE = 112;
|
||||
|
||||
export const ARC_FACE_5_LANDMARKS = [
|
||||
[38.2946, 51.6963],
|
||||
[73.5318, 51.5014],
|
||||
[56.0252, 71.7366],
|
||||
[41.5493, 92.3655],
|
||||
[70.7299, 92.2041],
|
||||
] as Array<[number, number]>;
|
||||
|
|
@ -1,5 +1,3 @@
|
|||
import * as tf from "@tensorflow/tfjs-core";
|
||||
|
||||
import { DebugInfo } from "hdbscan";
|
||||
import PQueue from "p-queue";
|
||||
import { EnteFile } from "types/file";
|
||||
|
|
@ -12,21 +10,9 @@ export interface MLSyncResult {
|
|||
nSyncedFaces: number;
|
||||
nFaceClusters: number;
|
||||
nFaceNoise: number;
|
||||
tsne?: any;
|
||||
error?: Error;
|
||||
}
|
||||
|
||||
export interface DebugFace {
|
||||
fileId: string;
|
||||
// face: FaceApiResult;
|
||||
face: AlignedFace;
|
||||
embedding: FaceEmbedding;
|
||||
faceImage: FaceImage;
|
||||
}
|
||||
|
||||
export declare type FaceImage = Array<Array<Array<number>>>;
|
||||
export declare type FaceImageBlob = Blob;
|
||||
|
||||
export declare type FaceDescriptor = Float32Array;
|
||||
|
||||
export declare type Cluster = Array<number>;
|
||||
|
|
@ -59,14 +45,7 @@ export declare type Landmark = Point;
|
|||
|
||||
export declare type ImageType = "Original" | "Preview";
|
||||
|
||||
export declare type FaceDetectionMethod =
|
||||
| "BlazeFace"
|
||||
| "FaceApiSSD"
|
||||
| "YoloFace";
|
||||
|
||||
export declare type ObjectDetectionMethod = "SSDMobileNetV2";
|
||||
|
||||
export declare type SceneDetectionMethod = "ImageScene";
|
||||
export declare type FaceDetectionMethod = "FaceApiSSD" | "YoloFace";
|
||||
|
||||
export declare type FaceCropMethod = "ArcFace";
|
||||
|
||||
|
|
@ -155,45 +134,15 @@ export interface Person {
|
|||
displayImageUrl?: string;
|
||||
}
|
||||
|
||||
export interface ObjectDetection {
|
||||
bbox: [number, number, number, number];
|
||||
class: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface DetectedObject {
|
||||
fileID: number;
|
||||
detection: ObjectDetection;
|
||||
}
|
||||
|
||||
export interface RealWorldObject extends DetectedObject {
|
||||
id: string;
|
||||
className: string;
|
||||
}
|
||||
|
||||
export interface Thing {
|
||||
id: number;
|
||||
name: string;
|
||||
files: Array<number>;
|
||||
}
|
||||
|
||||
export interface WordGroup {
|
||||
word: string;
|
||||
files: Array<number>;
|
||||
}
|
||||
|
||||
export interface MlFileData {
|
||||
fileId: number;
|
||||
faces?: Face[];
|
||||
objects?: RealWorldObject[];
|
||||
imageSource?: ImageType;
|
||||
imageDimensions?: Dimensions;
|
||||
faceDetectionMethod?: Versioned<FaceDetectionMethod>;
|
||||
faceCropMethod?: Versioned<FaceCropMethod>;
|
||||
faceAlignmentMethod?: Versioned<FaceAlignmentMethod>;
|
||||
faceEmbeddingMethod?: Versioned<FaceEmbeddingMethod>;
|
||||
objectDetectionMethod?: Versioned<ObjectDetectionMethod>;
|
||||
sceneDetectionMethod?: Versioned<SceneDetectionMethod>;
|
||||
mlVersion: number;
|
||||
errorCount: number;
|
||||
lastErrorMessage?: string;
|
||||
|
|
@ -203,17 +152,6 @@ export interface FaceDetectionConfig {
|
|||
method: FaceDetectionMethod;
|
||||
}
|
||||
|
||||
export interface ObjectDetectionConfig {
|
||||
method: ObjectDetectionMethod;
|
||||
maxNumBoxes: number;
|
||||
minScore: number;
|
||||
}
|
||||
|
||||
export interface SceneDetectionConfig {
|
||||
method: SceneDetectionMethod;
|
||||
minScore: number;
|
||||
}
|
||||
|
||||
export interface FaceCropConfig {
|
||||
enabled: boolean;
|
||||
method: FaceCropMethod;
|
||||
|
|
@ -263,9 +201,6 @@ export interface MLSyncConfig {
|
|||
blurDetection: BlurDetectionConfig;
|
||||
faceEmbedding: FaceEmbeddingConfig;
|
||||
faceClustering: FaceClusteringConfig;
|
||||
objectDetection: ObjectDetectionConfig;
|
||||
sceneDetection: SceneDetectionConfig;
|
||||
tsne?: TSNEConfig;
|
||||
mlVersion: number;
|
||||
}
|
||||
|
||||
|
|
@ -285,16 +220,12 @@ export interface MLSyncContext {
|
|||
faceEmbeddingService: FaceEmbeddingService;
|
||||
blurDetectionService: BlurDetectionService;
|
||||
faceClusteringService: ClusteringService;
|
||||
objectDetectionService: ObjectDetectionService;
|
||||
sceneDetectionService: SceneDetectionService;
|
||||
|
||||
localFilesMap: Map<number, EnteFile>;
|
||||
outOfSyncFiles: EnteFile[];
|
||||
nSyncedFiles: number;
|
||||
nSyncedFaces: number;
|
||||
allSyncedFacesMap?: Map<number, Array<Face>>;
|
||||
allSyncedObjectsMap?: Map<number, Array<RealWorldObject>>;
|
||||
tsne?: any;
|
||||
|
||||
error?: Error;
|
||||
|
||||
|
|
@ -314,7 +245,6 @@ export interface MLSyncFileContext {
|
|||
oldMlFile?: MlFileData;
|
||||
newMlFile?: MlFileData;
|
||||
|
||||
tfImage?: tf.Tensor3D;
|
||||
imageBitmap?: ImageBitmap;
|
||||
|
||||
newDetection?: boolean;
|
||||
|
|
@ -340,26 +270,6 @@ export interface FaceDetectionService {
|
|||
dispose(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ObjectDetectionService {
|
||||
method: Versioned<ObjectDetectionMethod>;
|
||||
// init(): Promise<void>;
|
||||
detectObjects(
|
||||
image: ImageBitmap,
|
||||
maxNumBoxes: number,
|
||||
minScore: number,
|
||||
): Promise<ObjectDetection[]>;
|
||||
dispose(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface SceneDetectionService {
|
||||
method: Versioned<SceneDetectionMethod>;
|
||||
// init(): Promise<void>;
|
||||
detectScenes(
|
||||
image: ImageBitmap,
|
||||
minScore: number,
|
||||
): Promise<ObjectDetection[]>;
|
||||
}
|
||||
|
||||
export interface FaceCropService {
|
||||
method: Versioned<FaceCropMethod>;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { FILE_TYPE } from "constants/file";
|
|||
import { City } from "services/locationSearchService";
|
||||
import { LocationTagData } from "types/entity";
|
||||
import { EnteFile } from "types/file";
|
||||
import { Person, Thing, WordGroup } from "types/machineLearning";
|
||||
import { Person } from "types/machineLearning";
|
||||
import { IndexStatus } from "types/machineLearning/ui";
|
||||
|
||||
export enum SuggestionType {
|
||||
|
|
@ -12,8 +12,6 @@ export enum SuggestionType {
|
|||
FILE_NAME = "FILE_NAME",
|
||||
PERSON = "PERSON",
|
||||
INDEX_STATUS = "INDEX_STATUS",
|
||||
THING = "THING",
|
||||
TEXT = "TEXT",
|
||||
FILE_CAPTION = "FILE_CAPTION",
|
||||
FILE_TYPE = "FILE_TYPE",
|
||||
CLIP = "CLIP",
|
||||
|
|
@ -34,8 +32,6 @@ export interface Suggestion {
|
|||
| number[]
|
||||
| Person
|
||||
| IndexStatus
|
||||
| Thing
|
||||
| WordGroup
|
||||
| LocationTagData
|
||||
| City
|
||||
| FILE_TYPE
|
||||
|
|
@ -50,8 +46,6 @@ export type Search = {
|
|||
collection?: number;
|
||||
files?: number[];
|
||||
person?: Person;
|
||||
thing?: Thing;
|
||||
text?: WordGroup;
|
||||
fileType?: FILE_TYPE;
|
||||
clip?: ClipSearchScores;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,34 +1,39 @@
|
|||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import { Matrix, inverse } from "ml-matrix";
|
||||
import { Matrix } from "ml-matrix";
|
||||
import { getSimilarityTransformation } from "similarity-transformation";
|
||||
import { Dimensions } from "types/image";
|
||||
import { FaceAlignment, FaceDetection } from "types/machineLearning";
|
||||
import {
|
||||
ARCFACE_LANDMARKS,
|
||||
ARCFACE_LANDMARKS_FACE_SIZE,
|
||||
ARC_FACE_5_LANDMARKS,
|
||||
} from "types/machineLearning/archface";
|
||||
import { cropWithRotation, transform } from "utils/image";
|
||||
import {
|
||||
computeRotation,
|
||||
enlargeBox,
|
||||
extractFaces,
|
||||
getBoxCenter,
|
||||
getBoxCenterPt,
|
||||
toTensor4D,
|
||||
} from ".";
|
||||
import { Box, Point } from "../../../thirdparty/face-api/classes";
|
||||
import { Point } from "../../../thirdparty/face-api/classes";
|
||||
|
||||
export function normalizeLandmarks(
|
||||
landmarks: Array<[number, number]>,
|
||||
faceSize: number,
|
||||
): Array<[number, number]> {
|
||||
return landmarks.map((landmark) =>
|
||||
landmark.map((p) => p / faceSize),
|
||||
) as Array<[number, number]>;
|
||||
const ARCFACE_LANDMARKS = [
|
||||
[38.2946, 51.6963],
|
||||
[73.5318, 51.5014],
|
||||
[56.0252, 71.7366],
|
||||
[56.1396, 92.2848],
|
||||
] as Array<[number, number]>;
|
||||
|
||||
const ARCFACE_LANDMARKS_FACE_SIZE = 112;
|
||||
|
||||
const ARC_FACE_5_LANDMARKS = [
|
||||
[38.2946, 51.6963],
|
||||
[73.5318, 51.5014],
|
||||
[56.0252, 71.7366],
|
||||
[41.5493, 92.3655],
|
||||
[70.7299, 92.2041],
|
||||
] as Array<[number, number]>;
|
||||
|
||||
export function getArcfaceAlignment(
|
||||
faceDetection: FaceDetection,
|
||||
): FaceAlignment {
|
||||
const landmarkCount = faceDetection.landmarks.length;
|
||||
return getFaceAlignmentUsingSimilarityTransform(
|
||||
faceDetection,
|
||||
normalizeLandmarks(
|
||||
landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS,
|
||||
ARCFACE_LANDMARKS_FACE_SIZE,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function getFaceAlignmentUsingSimilarityTransform(
|
||||
function getFaceAlignmentUsingSimilarityTransform(
|
||||
faceDetection: FaceDetection,
|
||||
alignedLandmarks: Array<[number, number]>,
|
||||
// alignmentMethod: Versioned<FaceAlignmentMethod>
|
||||
|
|
@ -72,175 +77,11 @@ export function getFaceAlignmentUsingSimilarityTransform(
|
|||
};
|
||||
}
|
||||
|
||||
export function getArcfaceAlignment(
|
||||
faceDetection: FaceDetection,
|
||||
): FaceAlignment {
|
||||
const landmarkCount = faceDetection.landmarks.length;
|
||||
return getFaceAlignmentUsingSimilarityTransform(
|
||||
faceDetection,
|
||||
normalizeLandmarks(
|
||||
landmarkCount === 5 ? ARC_FACE_5_LANDMARKS : ARCFACE_LANDMARKS,
|
||||
ARCFACE_LANDMARKS_FACE_SIZE,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function extractFaceImage(
|
||||
image: tf.Tensor4D,
|
||||
alignment: FaceAlignment,
|
||||
function normalizeLandmarks(
|
||||
landmarks: Array<[number, number]>,
|
||||
faceSize: number,
|
||||
) {
|
||||
const affineMat = new Matrix(alignment.affineMatrix);
|
||||
|
||||
const I = inverse(affineMat);
|
||||
|
||||
return tf.tidy(() => {
|
||||
const projection = tf.tensor2d([
|
||||
[
|
||||
I.get(0, 0),
|
||||
I.get(0, 1),
|
||||
I.get(0, 2),
|
||||
I.get(1, 0),
|
||||
I.get(1, 1),
|
||||
I.get(1, 2),
|
||||
0,
|
||||
0,
|
||||
],
|
||||
]);
|
||||
const faceImage = tf.image.transform(
|
||||
image,
|
||||
projection,
|
||||
"bilinear",
|
||||
"constant",
|
||||
0,
|
||||
[faceSize, faceSize],
|
||||
);
|
||||
return faceImage;
|
||||
});
|
||||
}
|
||||
|
||||
export function tfExtractFaceImages(
|
||||
image: tf.Tensor3D | tf.Tensor4D,
|
||||
alignments: Array<FaceAlignment>,
|
||||
faceSize: number,
|
||||
): tf.Tensor4D {
|
||||
return tf.tidy(() => {
|
||||
const tf4dFloat32Image = toTensor4D(image, "float32");
|
||||
const faceImages = new Array<tf.Tensor3D>(alignments.length);
|
||||
for (let i = 0; i < alignments.length; i++) {
|
||||
faceImages[i] = tf.squeeze(
|
||||
extractFaceImage(tf4dFloat32Image, alignments[i], faceSize),
|
||||
[0],
|
||||
);
|
||||
}
|
||||
|
||||
return tf.stack(faceImages) as tf.Tensor4D;
|
||||
});
|
||||
}
|
||||
|
||||
export function getAlignedFaceBox(alignment: FaceAlignment) {
|
||||
return new Box({
|
||||
x: alignment.center.x - alignment.size / 2,
|
||||
y: alignment.center.y - alignment.size / 2,
|
||||
width: alignment.size,
|
||||
height: alignment.size,
|
||||
}).round();
|
||||
}
|
||||
|
||||
export function ibExtractFaceImage(
|
||||
image: ImageBitmap,
|
||||
alignment: FaceAlignment,
|
||||
faceSize: number,
|
||||
): ImageBitmap {
|
||||
const box = getAlignedFaceBox(alignment);
|
||||
const faceSizeDimentions: Dimensions = {
|
||||
width: faceSize,
|
||||
height: faceSize,
|
||||
};
|
||||
return cropWithRotation(
|
||||
image,
|
||||
box,
|
||||
alignment.rotation,
|
||||
faceSizeDimentions,
|
||||
faceSizeDimentions,
|
||||
);
|
||||
}
|
||||
|
||||
// Used in MLDebugViewOnly
|
||||
export function ibExtractFaceImageUsingTransform(
|
||||
image: ImageBitmap,
|
||||
alignment: FaceAlignment,
|
||||
faceSize: number,
|
||||
): ImageBitmap {
|
||||
const scaledMatrix = new Matrix(alignment.affineMatrix)
|
||||
.mul(faceSize)
|
||||
.to2DArray();
|
||||
// log.info("scaledMatrix: ", scaledMatrix);
|
||||
return transform(image, scaledMatrix, faceSize, faceSize);
|
||||
}
|
||||
|
||||
export function ibExtractFaceImages(
|
||||
image: ImageBitmap,
|
||||
alignments: Array<FaceAlignment>,
|
||||
faceSize: number,
|
||||
): Array<ImageBitmap> {
|
||||
return alignments.map((alignment) =>
|
||||
ibExtractFaceImage(image, alignment, faceSize),
|
||||
);
|
||||
}
|
||||
|
||||
const BLAZEFACE_LEFT_EYE_INDEX = 0;
|
||||
const BLAZEFACE_RIGHT_EYE_INDEX = 1;
|
||||
// const BLAZEFACE_NOSE_INDEX = 2;
|
||||
const BLAZEFACE_MOUTH_INDEX = 3;
|
||||
|
||||
export function getRotatedFaceImage(
|
||||
image: tf.Tensor3D | tf.Tensor4D,
|
||||
faceDetection: FaceDetection,
|
||||
padding: number = 1.5,
|
||||
): tf.Tensor4D {
|
||||
const paddedBox = enlargeBox(faceDetection.box, padding);
|
||||
// log.info("paddedBox", paddedBox);
|
||||
const landmarkPoints = faceDetection.landmarks;
|
||||
|
||||
return tf.tidy(() => {
|
||||
const tf4dFloat32Image = toTensor4D(image, "float32");
|
||||
let angle = 0;
|
||||
const leftEye = landmarkPoints[BLAZEFACE_LEFT_EYE_INDEX];
|
||||
const rightEye = landmarkPoints[BLAZEFACE_RIGHT_EYE_INDEX];
|
||||
const foreheadCenter = getBoxCenterPt(leftEye, rightEye);
|
||||
|
||||
angle = computeRotation(
|
||||
landmarkPoints[BLAZEFACE_MOUTH_INDEX],
|
||||
foreheadCenter,
|
||||
); // landmarkPoints[BLAZEFACE_NOSE_INDEX]
|
||||
// angle = computeRotation(leftEye, rightEye);
|
||||
// log.info('angle: ', angle);
|
||||
|
||||
const faceCenter = getBoxCenter(faceDetection.box);
|
||||
// log.info('faceCenter: ', faceCenter);
|
||||
const faceCenterNormalized: [number, number] = [
|
||||
faceCenter.x / tf4dFloat32Image.shape[2],
|
||||
faceCenter.y / tf4dFloat32Image.shape[1],
|
||||
];
|
||||
// log.info('faceCenterNormalized: ', faceCenterNormalized);
|
||||
|
||||
let rotatedImage = tf4dFloat32Image;
|
||||
if (angle !== 0) {
|
||||
rotatedImage = tf.image.rotateWithOffset(
|
||||
tf4dFloat32Image,
|
||||
angle,
|
||||
0,
|
||||
faceCenterNormalized,
|
||||
);
|
||||
}
|
||||
|
||||
const faceImageTensor = extractFaces(
|
||||
rotatedImage,
|
||||
[paddedBox],
|
||||
paddedBox.width > 224 ? 448 : 224,
|
||||
);
|
||||
return faceImageTensor;
|
||||
// return tf.gather(faceImageTensor, 0);
|
||||
});
|
||||
): Array<[number, number]> {
|
||||
return landmarks.map((landmark) =>
|
||||
landmark.map((p) => p / faceSize),
|
||||
) as Array<[number, number]>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,15 @@
|
|||
import log from "@/next/log";
|
||||
import { CacheStorageService } from "@ente/shared/storage/cacheStorage";
|
||||
import { CACHES } from "@ente/shared/storage/cacheStorage/constants";
|
||||
import { getBlobFromCache } from "@ente/shared/storage/cacheStorage/helpers";
|
||||
import { compose, Matrix, scale, translate } from "transformation-matrix";
|
||||
import { BlobOptions, Dimensions } from "types/image";
|
||||
import { BlobOptions } from "types/image";
|
||||
import {
|
||||
AlignedFace,
|
||||
FaceAlignment,
|
||||
FaceCrop,
|
||||
FaceCropConfig,
|
||||
FaceDetection,
|
||||
MlFileData,
|
||||
StoredFaceCrop,
|
||||
} from "types/machineLearning";
|
||||
import { cropWithRotation, imageBitmapToBlob } from "utils/image";
|
||||
import { enlargeBox } from ".";
|
||||
import { Box } from "../../../thirdparty/face-api/classes";
|
||||
import { getAlignedFaceBox } from "./faceAlign";
|
||||
import { transformBox, transformPoints } from "./transform";
|
||||
|
||||
export function getFaceCrop(
|
||||
imageBitmap: ImageBitmap,
|
||||
|
|
@ -38,7 +30,25 @@ export function getFaceCrop(
|
|||
};
|
||||
}
|
||||
|
||||
export async function storeFaceCropForBlob(
|
||||
function getAlignedFaceBox(alignment: FaceAlignment) {
|
||||
return new Box({
|
||||
x: alignment.center.x - alignment.size / 2,
|
||||
y: alignment.center.y - alignment.size / 2,
|
||||
width: alignment.size,
|
||||
height: alignment.size,
|
||||
}).round();
|
||||
}
|
||||
|
||||
export async function storeFaceCrop(
|
||||
faceId: string,
|
||||
faceCrop: FaceCrop,
|
||||
blobOptions: BlobOptions,
|
||||
): Promise<StoredFaceCrop> {
|
||||
const faceCropBlob = await imageBitmapToBlob(faceCrop.image, blobOptions);
|
||||
return storeFaceCropForBlob(faceId, faceCrop.imageBox, faceCropBlob);
|
||||
}
|
||||
|
||||
async function storeFaceCropForBlob(
|
||||
faceId: string,
|
||||
imageBox: Box,
|
||||
faceCropBlob: Blob,
|
||||
|
|
@ -52,166 +62,3 @@ export async function storeFaceCropForBlob(
|
|||
imageBox: imageBox,
|
||||
};
|
||||
}
|
||||
|
||||
export async function storeFaceCrop(
|
||||
faceId: string,
|
||||
faceCrop: FaceCrop,
|
||||
blobOptions: BlobOptions,
|
||||
): Promise<StoredFaceCrop> {
|
||||
const faceCropBlob = await imageBitmapToBlob(faceCrop.image, blobOptions);
|
||||
return storeFaceCropForBlob(faceId, faceCrop.imageBox, faceCropBlob);
|
||||
}
|
||||
|
||||
export async function getFaceCropBlobFromStorage(
|
||||
storedFaceCrop: StoredFaceCrop,
|
||||
): Promise<Blob> {
|
||||
return getBlobFromCache(CACHES.FACE_CROPS, storedFaceCrop.imageUrl);
|
||||
}
|
||||
|
||||
export async function getFaceCropFromStorage(
|
||||
storedFaceCrop: StoredFaceCrop,
|
||||
): Promise<FaceCrop> {
|
||||
const faceCropBlob = await getFaceCropBlobFromStorage(storedFaceCrop);
|
||||
const faceCropImage = await createImageBitmap(faceCropBlob);
|
||||
|
||||
return {
|
||||
image: faceCropImage,
|
||||
imageBox: storedFaceCrop.imageBox,
|
||||
};
|
||||
}
|
||||
|
||||
export async function removeOldFaceCrops(
|
||||
oldMLFileData: MlFileData,
|
||||
newMLFileData: MlFileData,
|
||||
) {
|
||||
const newFaceCropUrls =
|
||||
newMLFileData?.faces
|
||||
?.map((f) => f.crop?.imageUrl)
|
||||
?.filter((fc) => fc !== null && fc !== undefined) || [];
|
||||
|
||||
const oldFaceCropUrls =
|
||||
oldMLFileData?.faces
|
||||
?.map((f) => f.crop?.imageUrl)
|
||||
?.filter((fc) => fc !== null && fc !== undefined) || [];
|
||||
|
||||
const unusedFaceCropUrls = oldFaceCropUrls.filter(
|
||||
(oldUrl) => !newFaceCropUrls.includes(oldUrl),
|
||||
);
|
||||
if (!unusedFaceCropUrls || unusedFaceCropUrls.length < 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
return removeFaceCropUrls(unusedFaceCropUrls);
|
||||
}
|
||||
|
||||
export async function removeFaceCropUrls(faceCropUrls: Array<string>) {
|
||||
log.info("Removing face crop urls: ", JSON.stringify(faceCropUrls));
|
||||
const faceCropCache = await CacheStorageService.open(CACHES.FACE_CROPS);
|
||||
const urlRemovalPromises = faceCropUrls?.map((url) =>
|
||||
faceCropCache.delete(url),
|
||||
);
|
||||
return urlRemovalPromises && Promise.all(urlRemovalPromises);
|
||||
}
|
||||
|
||||
export function extractFaceImageFromCrop(
|
||||
faceCrop: FaceCrop,
|
||||
box: Box,
|
||||
rotation: number,
|
||||
faceSize: number,
|
||||
): ImageBitmap {
|
||||
const faceCropImage = faceCrop?.image;
|
||||
let imageBox = faceCrop?.imageBox;
|
||||
if (!faceCropImage || !imageBox) {
|
||||
throw Error("Face crop not present");
|
||||
}
|
||||
|
||||
// TODO: Have better serialization to avoid creating new object manually when calling class methods
|
||||
imageBox = new Box(imageBox);
|
||||
const scale = faceCropImage.width / imageBox.width;
|
||||
const transformedBox = box
|
||||
.shift(-imageBox.x, -imageBox.y)
|
||||
.rescale(scale)
|
||||
.round();
|
||||
// log.info({ box, imageBox, faceCropImage, scale, scaledBox, scaledImageBox, shiftedBox });
|
||||
|
||||
const faceSizeDimentions: Dimensions = {
|
||||
width: faceSize,
|
||||
height: faceSize,
|
||||
};
|
||||
const faceImage = cropWithRotation(
|
||||
faceCropImage,
|
||||
transformedBox,
|
||||
rotation,
|
||||
faceSizeDimentions,
|
||||
faceSizeDimentions,
|
||||
);
|
||||
|
||||
return faceImage;
|
||||
}
|
||||
|
||||
export async function ibExtractFaceImageFromCrop(
|
||||
faceCrop: FaceCrop,
|
||||
alignment: FaceAlignment,
|
||||
faceSize: number,
|
||||
): Promise<ImageBitmap> {
|
||||
const box = getAlignedFaceBox(alignment);
|
||||
|
||||
return extractFaceImageFromCrop(
|
||||
faceCrop,
|
||||
box,
|
||||
alignment.rotation,
|
||||
faceSize,
|
||||
);
|
||||
}
|
||||
|
||||
export async function ibExtractFaceImagesFromCrops(
|
||||
faces: Array<AlignedFace>,
|
||||
faceSize: number,
|
||||
): Promise<Array<ImageBitmap>> {
|
||||
const faceImagePromises = faces.map(async (alignedFace) => {
|
||||
const faceCrop = await getFaceCropFromStorage(alignedFace.crop);
|
||||
return ibExtractFaceImageFromCrop(
|
||||
faceCrop,
|
||||
alignedFace.alignment,
|
||||
faceSize,
|
||||
);
|
||||
});
|
||||
return Promise.all(faceImagePromises);
|
||||
}
|
||||
|
||||
export function transformFace(faceDetection: FaceDetection, transform: Matrix) {
|
||||
return {
|
||||
...faceDetection,
|
||||
|
||||
box: transformBox(faceDetection.box, transform),
|
||||
landmarks: transformPoints(faceDetection.landmarks, transform),
|
||||
};
|
||||
}
|
||||
|
||||
export function transformToFaceCropDims(
|
||||
faceCrop: FaceCrop,
|
||||
faceDetection: FaceDetection,
|
||||
) {
|
||||
const imageBox = new Box(faceCrop.imageBox);
|
||||
|
||||
const transform = compose(
|
||||
scale(faceCrop.image.width / imageBox.width),
|
||||
translate(-imageBox.x, -imageBox.y),
|
||||
);
|
||||
|
||||
return transformFace(faceDetection, transform);
|
||||
}
|
||||
|
||||
export function transformToImageDims(
|
||||
faceCrop: FaceCrop,
|
||||
faceDetection: FaceDetection,
|
||||
) {
|
||||
const imageBox = new Box(faceCrop.imageBox);
|
||||
|
||||
const transform = compose(
|
||||
translate(imageBox.x, imageBox.y),
|
||||
scale(imageBox.width / faceCrop.image.width),
|
||||
);
|
||||
|
||||
return transformFace(faceDetection, transform);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,108 +0,0 @@
|
|||
import { euclidean } from "hdbscan";
|
||||
import { FaceDetection } from "types/machineLearning";
|
||||
import { getNearestPointIndex, newBox } from ".";
|
||||
import { Box, Point } from "../../../thirdparty/face-api/classes";
|
||||
import {
|
||||
computeTransformToBox,
|
||||
transformBox,
|
||||
transformPoints,
|
||||
} from "./transform";
|
||||
|
||||
export function transformPaddedToImage(
|
||||
detection: FaceDetection,
|
||||
faceImage: ImageBitmap,
|
||||
imageBox: Box,
|
||||
paddedBox: Box,
|
||||
) {
|
||||
const inBox = newBox(0, 0, faceImage.width, faceImage.height);
|
||||
imageBox.x = paddedBox.x;
|
||||
imageBox.y = paddedBox.y;
|
||||
const transform = computeTransformToBox(inBox, imageBox);
|
||||
|
||||
detection.box = transformBox(detection.box, transform);
|
||||
detection.landmarks = transformPoints(detection.landmarks, transform);
|
||||
}
|
||||
|
||||
export function getDetectionCenter(detection: FaceDetection) {
|
||||
const center = new Point(0, 0);
|
||||
// TODO: first 4 landmarks is applicable to blazeface only
|
||||
// this needs to consider eyes, nose and mouth landmarks to take center
|
||||
detection.landmarks?.slice(0, 4).forEach((p) => {
|
||||
center.x += p.x;
|
||||
center.y += p.y;
|
||||
});
|
||||
|
||||
return center.div({ x: 4, y: 4 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the nearest face detection from a list of detections to a specified detection.
|
||||
*
|
||||
* This function calculates the center of each detection and then finds the detection whose center is nearest to the center of the specified detection.
|
||||
* If a maximum distance is specified, only detections within that distance are considered.
|
||||
*
|
||||
* @param toDetection - The face detection to find the nearest detection to.
|
||||
* @param fromDetections - An array of face detections to search in.
|
||||
* @param maxDistance - The maximum distance between the centers of the two detections for a detection to be considered. If not specified, all detections are considered.
|
||||
*
|
||||
* @returns The nearest face detection from the list, or `undefined` if no detection is within the maximum distance.
|
||||
*/
|
||||
export function getNearestDetection(
|
||||
toDetection: FaceDetection,
|
||||
fromDetections: Array<FaceDetection>,
|
||||
maxDistance?: number,
|
||||
) {
|
||||
const toCenter = getDetectionCenter(toDetection);
|
||||
const centers = fromDetections.map((d) => getDetectionCenter(d));
|
||||
const nearestIndex = getNearestPointIndex(toCenter, centers, maxDistance);
|
||||
|
||||
return nearestIndex >= 0 && fromDetections[nearestIndex];
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes duplicate face detections from an array of detections.
|
||||
*
|
||||
* This function sorts the detections by their probability in descending order, then iterates over them.
|
||||
* For each detection, it calculates the Euclidean distance to all other detections.
|
||||
* If the distance is less than or equal to the specified threshold (`withinDistance`), the other detection is considered a duplicate and is removed.
|
||||
*
|
||||
* @param detections - An array of face detections to remove duplicates from.
|
||||
* @param withinDistance - The maximum Euclidean distance between two detections for them to be considered duplicates.
|
||||
*
|
||||
* @returns An array of face detections with duplicates removed.
|
||||
*/
|
||||
export function removeDuplicateDetections(
|
||||
detections: Array<FaceDetection>,
|
||||
withinDistance: number,
|
||||
) {
|
||||
// console.time('removeDuplicates');
|
||||
detections.sort((a, b) => b.probability - a.probability);
|
||||
const isSelected = new Map<number, boolean>();
|
||||
for (let i = 0; i < detections.length; i++) {
|
||||
if (isSelected.get(i) === false) {
|
||||
continue;
|
||||
}
|
||||
isSelected.set(i, true);
|
||||
for (let j = i + 1; j < detections.length; j++) {
|
||||
if (isSelected.get(j) === false) {
|
||||
continue;
|
||||
}
|
||||
const centeri = getDetectionCenter(detections[i]);
|
||||
const centerj = getDetectionCenter(detections[j]);
|
||||
const dist = euclidean(
|
||||
[centeri.x, centeri.y],
|
||||
[centerj.x, centerj.y],
|
||||
);
|
||||
if (dist <= withinDistance) {
|
||||
isSelected.set(j, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const uniques: Array<FaceDetection> = [];
|
||||
for (let i = 0; i < detections.length; i++) {
|
||||
isSelected.get(i) && uniques.push(detections[i]);
|
||||
}
|
||||
// console.timeEnd('removeDuplicates');
|
||||
return uniques;
|
||||
}
|
||||
|
|
@ -1,11 +1,7 @@
|
|||
import log from "@/next/log";
|
||||
import { CACHES } from "@ente/shared/storage/cacheStorage/constants";
|
||||
import { cached } from "@ente/shared/storage/cacheStorage/helpers";
|
||||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import { NormalizedFace } from "blazeface-back";
|
||||
import { FILE_TYPE } from "constants/file";
|
||||
import { BLAZEFACE_FACE_SIZE } from "constants/mlConfig";
|
||||
import { euclidean } from "hdbscan";
|
||||
import PQueue from "p-queue";
|
||||
import DownloadManager from "services/download";
|
||||
import { getLocalFiles } from "services/fileService";
|
||||
|
|
@ -13,152 +9,22 @@ import { decodeLivePhoto } from "services/livePhotoService";
|
|||
import { EnteFile } from "types/file";
|
||||
import { Dimensions } from "types/image";
|
||||
import {
|
||||
AlignedFace,
|
||||
DetectedFace,
|
||||
DetectedObject,
|
||||
Face,
|
||||
FaceAlignment,
|
||||
FaceImageBlob,
|
||||
MlFileData,
|
||||
Person,
|
||||
RealWorldObject,
|
||||
Versioned,
|
||||
} from "types/machineLearning";
|
||||
import { getRenderableImage } from "utils/file";
|
||||
import { clamp, imageBitmapToBlob, warpAffineFloat32List } from "utils/image";
|
||||
import { clamp, warpAffineFloat32List } from "utils/image";
|
||||
import mlIDbStorage from "utils/storage/mlIDbStorage";
|
||||
import { Box, Point } from "../../../thirdparty/face-api/classes";
|
||||
import { ibExtractFaceImage, ibExtractFaceImages } from "./faceAlign";
|
||||
import { getFaceCropBlobFromStorage } from "./faceCrop";
|
||||
|
||||
export function f32Average(descriptors: Float32Array[]) {
|
||||
if (descriptors.length < 1) {
|
||||
throw Error("f32Average: input size 0");
|
||||
}
|
||||
|
||||
if (descriptors.length === 1) {
|
||||
return descriptors[0];
|
||||
}
|
||||
|
||||
const f32Size = descriptors[0].length;
|
||||
const avg = new Float32Array(f32Size);
|
||||
|
||||
for (let index = 0; index < f32Size; index++) {
|
||||
avg[index] = descriptors[0][index];
|
||||
for (let desc = 1; desc < descriptors.length; desc++) {
|
||||
avg[index] = avg[index] + descriptors[desc][index];
|
||||
}
|
||||
avg[index] = avg[index] / descriptors.length;
|
||||
}
|
||||
|
||||
return avg;
|
||||
}
|
||||
|
||||
export function isTensor(tensor: any, dim: number) {
|
||||
return tensor instanceof tf.Tensor && tensor.shape.length === dim;
|
||||
}
|
||||
|
||||
export function isTensor1D(tensor: any): tensor is tf.Tensor1D {
|
||||
return isTensor(tensor, 1);
|
||||
}
|
||||
|
||||
export function isTensor2D(tensor: any): tensor is tf.Tensor2D {
|
||||
return isTensor(tensor, 2);
|
||||
}
|
||||
|
||||
export function isTensor3D(tensor: any): tensor is tf.Tensor3D {
|
||||
return isTensor(tensor, 3);
|
||||
}
|
||||
|
||||
export function isTensor4D(tensor: any): tensor is tf.Tensor4D {
|
||||
return isTensor(tensor, 4);
|
||||
}
|
||||
|
||||
export function toTensor4D(
|
||||
image: tf.Tensor3D | tf.Tensor4D,
|
||||
dtype?: tf.DataType,
|
||||
) {
|
||||
return tf.tidy(() => {
|
||||
let reshapedImage: tf.Tensor4D;
|
||||
if (isTensor3D(image)) {
|
||||
reshapedImage = tf.expandDims(image, 0);
|
||||
} else if (isTensor4D(image)) {
|
||||
reshapedImage = image;
|
||||
} else {
|
||||
throw Error("toTensor4D only supports Tensor3D and Tensor4D input");
|
||||
}
|
||||
if (dtype) {
|
||||
reshapedImage = tf.cast(reshapedImage, dtype);
|
||||
}
|
||||
|
||||
return reshapedImage;
|
||||
});
|
||||
}
|
||||
|
||||
export function imageBitmapsToTensor4D(imageBitmaps: Array<ImageBitmap>) {
|
||||
return tf.tidy(() => {
|
||||
const tfImages = imageBitmaps.map((ib) => tf.browser.fromPixels(ib));
|
||||
return tf.stack(tfImages) as tf.Tensor4D;
|
||||
});
|
||||
}
|
||||
|
||||
export function extractFaces(
|
||||
image: tf.Tensor3D | tf.Tensor4D,
|
||||
facebBoxes: Array<Box>,
|
||||
faceSize: number,
|
||||
) {
|
||||
return tf.tidy(() => {
|
||||
const reshapedImage = toTensor4D(image, "float32");
|
||||
|
||||
const boxes = facebBoxes.map((box) => {
|
||||
const normalized = box.rescale({
|
||||
width: 1 / reshapedImage.shape[2],
|
||||
height: 1 / reshapedImage.shape[1],
|
||||
});
|
||||
|
||||
return [
|
||||
normalized.top,
|
||||
normalized.left,
|
||||
normalized.bottom,
|
||||
normalized.right,
|
||||
];
|
||||
});
|
||||
|
||||
// log.info('boxes: ', boxes[0]);
|
||||
|
||||
const faceImagesTensor = tf.image.cropAndResize(
|
||||
reshapedImage,
|
||||
boxes,
|
||||
tf.fill([boxes.length], 0, "int32"),
|
||||
[faceSize, faceSize],
|
||||
);
|
||||
|
||||
return faceImagesTensor;
|
||||
});
|
||||
}
|
||||
|
||||
export function newBox(x: number, y: number, width: number, height: number) {
|
||||
return new Box({ x, y, width, height });
|
||||
}
|
||||
|
||||
export function newBoxFromPoints(
|
||||
left: number,
|
||||
top: number,
|
||||
right: number,
|
||||
bottom: number,
|
||||
) {
|
||||
return new Box({ left, top, right, bottom });
|
||||
}
|
||||
|
||||
export function normFaceBox(face: NormalizedFace) {
|
||||
return newBoxFromPoints(
|
||||
face.topLeft[0],
|
||||
face.topLeft[1],
|
||||
face.bottomRight[0],
|
||||
face.bottomRight[1],
|
||||
);
|
||||
}
|
||||
|
||||
export function getBoxCenterPt(topLeft: Point, bottomRight: Point): Point {
|
||||
return topLeft.add(bottomRight.sub(topLeft).div(new Point(2, 2)));
|
||||
}
|
||||
|
|
@ -180,74 +46,17 @@ export function enlargeBox(box: Box, factor: number = 1.5) {
|
|||
});
|
||||
}
|
||||
|
||||
export function normalizeRadians(angle: number) {
|
||||
return angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI));
|
||||
}
|
||||
|
||||
export function computeRotation(point1: Point, point2: Point) {
|
||||
const radians =
|
||||
Math.PI / 2 - Math.atan2(-(point2.y - point1.y), point2.x - point1.x);
|
||||
return normalizeRadians(radians);
|
||||
}
|
||||
|
||||
export function getAllFacesFromMap(allFacesMap: Map<number, Array<Face>>) {
|
||||
const allFaces = [...allFacesMap.values()].flat();
|
||||
|
||||
return allFaces;
|
||||
}
|
||||
|
||||
export function getAllObjectsFromMap(
|
||||
allObjectsMap: Map<number, Array<RealWorldObject>>,
|
||||
) {
|
||||
return [...allObjectsMap.values()].flat();
|
||||
}
|
||||
|
||||
export async function getLocalFile(fileId: number) {
|
||||
const localFiles = await getLocalFiles();
|
||||
return localFiles.find((f) => f.id === fileId);
|
||||
}
|
||||
|
||||
export async function getFaceImage(
|
||||
face: AlignedFace,
|
||||
token: string,
|
||||
faceSize: number = BLAZEFACE_FACE_SIZE,
|
||||
file?: EnteFile,
|
||||
): Promise<FaceImageBlob> {
|
||||
if (!file) {
|
||||
file = await getLocalFile(face.fileId);
|
||||
}
|
||||
|
||||
const imageBitmap = await getOriginalImageBitmap(file);
|
||||
const faceImageBitmap = ibExtractFaceImage(
|
||||
imageBitmap,
|
||||
face.alignment,
|
||||
faceSize,
|
||||
);
|
||||
const faceImage = imageBitmapToBlob(faceImageBitmap);
|
||||
faceImageBitmap.close();
|
||||
imageBitmap.close();
|
||||
|
||||
return faceImage;
|
||||
}
|
||||
|
||||
export async function extractFaceImages(
|
||||
faces: Array<AlignedFace>,
|
||||
faceSize: number,
|
||||
image?: ImageBitmap,
|
||||
) {
|
||||
// if (faces.length === faces.filter((f) => f.crop).length) {
|
||||
// return ibExtractFaceImagesFromCrops(faces, faceSize);
|
||||
// } else
|
||||
if (image) {
|
||||
const faceAlignments = faces.map((f) => f.alignment);
|
||||
return ibExtractFaceImages(image, faceAlignments, faceSize);
|
||||
} else {
|
||||
throw Error(
|
||||
"Either face crops or image is required to extract face images",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function extractFaceImagesToFloat32(
|
||||
faceAlignments: Array<FaceAlignment>,
|
||||
faceSize: number,
|
||||
|
|
@ -270,10 +79,6 @@ export async function extractFaceImagesToFloat32(
|
|||
return faceData;
|
||||
}
|
||||
|
||||
export function leftFillNum(num: number, length: number, padding: number) {
|
||||
return num.toString().padStart(length, padding.toString());
|
||||
}
|
||||
|
||||
export function getFaceId(detectedFace: DetectedFace, imageDims: Dimensions) {
|
||||
const xMin = clamp(
|
||||
detectedFace.detection.box.x / imageDims.width,
|
||||
|
|
@ -312,45 +117,10 @@ export function getFaceId(detectedFace: DetectedFace, imageDims: Dimensions) {
|
|||
return faceID;
|
||||
}
|
||||
|
||||
export function getObjectId(
|
||||
detectedObject: DetectedObject,
|
||||
imageDims: Dimensions,
|
||||
) {
|
||||
const imgDimPoint = new Point(imageDims.width, imageDims.height);
|
||||
const objectCenterPoint = new Point(
|
||||
detectedObject.detection.bbox[2] / 2,
|
||||
detectedObject.detection.bbox[3] / 2,
|
||||
);
|
||||
const gridPt = objectCenterPoint
|
||||
.mul(new Point(100, 100))
|
||||
.div(imgDimPoint)
|
||||
.floor()
|
||||
.bound(0, 99);
|
||||
const gridPaddedX = leftFillNum(gridPt.x, 2, 0);
|
||||
const gridPaddedY = leftFillNum(gridPt.y, 2, 0);
|
||||
|
||||
return `${detectedObject.fileID}-${gridPaddedX}-${gridPaddedY}`;
|
||||
}
|
||||
|
||||
export async function getTFImage(blob): Promise<tf.Tensor3D> {
|
||||
const imageBitmap = await createImageBitmap(blob);
|
||||
const tfImage = tf.browser.fromPixels(imageBitmap);
|
||||
imageBitmap.close();
|
||||
|
||||
return tfImage;
|
||||
}
|
||||
|
||||
export async function getImageBlobBitmap(blob: Blob): Promise<ImageBitmap> {
|
||||
return await createImageBitmap(blob);
|
||||
}
|
||||
|
||||
// export async function getTFImageUsingJpegJS(blob: Blob): Promise<TFImageBitmap> {
|
||||
// const imageData = jpegjs.decode(await blob.arrayBuffer());
|
||||
// const tfImage = tf.browser.fromPixels(imageData);
|
||||
|
||||
// return new TFImageBitmap(undefined, tfImage);
|
||||
// }
|
||||
|
||||
async function getOriginalFile(file: EnteFile, queue?: PQueue) {
|
||||
let fileStream;
|
||||
if (queue) {
|
||||
|
|
@ -453,21 +223,6 @@ export async function getUnidentifiedFaces(
|
|||
);
|
||||
}
|
||||
|
||||
export async function getFaceCropBlobs(
|
||||
faces: Array<Face>,
|
||||
): Promise<Array<FaceImageBlob>> {
|
||||
const faceCrops = faces
|
||||
.map((f) => f.crop)
|
||||
.filter((faceCrop) => faceCrop !== null && faceCrop !== undefined);
|
||||
|
||||
return (
|
||||
faceCrops &&
|
||||
Promise.all(
|
||||
faceCrops.map((faceCrop) => getFaceCropBlobFromStorage(faceCrop)),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export async function getAllPeople(limit: number = undefined) {
|
||||
let people: Array<Person> = await mlIDbStorage.getAllPeople();
|
||||
// await mlPeopleStore.iterate<Person, void>((person) => {
|
||||
|
|
@ -531,27 +286,6 @@ export function areFaceIdsSame(ofFaces: Array<Face>, toFaces: Array<Face>) {
|
|||
);
|
||||
}
|
||||
|
||||
export function getNearestPointIndex(
|
||||
toPoint: Point,
|
||||
fromPoints: Array<Point>,
|
||||
maxDistance?: number,
|
||||
) {
|
||||
const dists = fromPoints.map((point, i) => ({
|
||||
index: i,
|
||||
point: point,
|
||||
distance: euclidean([point.x, point.y], [toPoint.x, toPoint.y]),
|
||||
}));
|
||||
const nearest = findFirstIfSorted(
|
||||
dists,
|
||||
(a, b) => Math.abs(a.distance) - Math.abs(b.distance),
|
||||
);
|
||||
|
||||
// log.info('Nearest dist: ', nearest.distance, maxDistance);
|
||||
if (!maxDistance || nearest.distance <= maxDistance) {
|
||||
return nearest.index;
|
||||
}
|
||||
}
|
||||
|
||||
export function logQueueStats(queue: PQueue, name: string) {
|
||||
queue.on("active", () =>
|
||||
log.info(
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
import { newBoxFromPoints } from ".";
|
||||
import { Box, Point } from "../../../thirdparty/face-api/classes";
|
||||
|
||||
import {
|
||||
Matrix,
|
||||
applyToPoint,
|
||||
compose,
|
||||
scale,
|
||||
translate,
|
||||
} from "transformation-matrix";
|
||||
|
||||
export function computeTransformToBox(inBox: Box, toBox: Box): Matrix {
|
||||
return compose(
|
||||
translate(toBox.x, toBox.y),
|
||||
scale(toBox.width / inBox.width, toBox.height / inBox.height),
|
||||
);
|
||||
}
|
||||
|
||||
export function transformPoint(point: Point, transform: Matrix) {
|
||||
const txdPoint = applyToPoint(transform, point);
|
||||
return new Point(txdPoint.x, txdPoint.y);
|
||||
}
|
||||
|
||||
export function transformPoints(points: Point[], transform: Matrix) {
|
||||
return points?.map((p) => transformPoint(p, transform));
|
||||
}
|
||||
|
||||
export function transformBox(box: Box, transform: Matrix) {
|
||||
const topLeft = transformPoint(box.topLeft, transform);
|
||||
const bottomRight = transformPoint(box.bottomRight, transform);
|
||||
|
||||
return newBoxFromPoints(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y);
|
||||
}
|
||||
|
|
@ -15,14 +15,7 @@ import {
|
|||
openDB,
|
||||
} from "idb";
|
||||
import isElectron from "is-electron";
|
||||
import {
|
||||
Face,
|
||||
MLLibraryData,
|
||||
MlFileData,
|
||||
Person,
|
||||
RealWorldObject,
|
||||
Thing,
|
||||
} from "types/machineLearning";
|
||||
import { Face, MLLibraryData, MlFileData, Person } from "types/machineLearning";
|
||||
import { IndexStatus } from "types/machineLearning/ui";
|
||||
|
||||
interface Config {}
|
||||
|
|
@ -42,9 +35,11 @@ interface MLDb extends DBSchema {
|
|||
key: number;
|
||||
value: Person;
|
||||
};
|
||||
// Unused, we only retain this is the schema so that we can delete it during
|
||||
// migration.
|
||||
things: {
|
||||
key: number;
|
||||
value: Thing;
|
||||
value: unknown;
|
||||
};
|
||||
versions: {
|
||||
key: string;
|
||||
|
|
@ -72,7 +67,7 @@ class MLIDbStorage {
|
|||
}
|
||||
|
||||
private openDB(): Promise<IDBPDatabase<MLDb>> {
|
||||
return openDB<MLDb>(MLDATA_DB_NAME, 3, {
|
||||
return openDB<MLDb>(MLDATA_DB_NAME, 4, {
|
||||
terminated: async () => {
|
||||
log.error("ML Indexed DB terminated");
|
||||
this._db = undefined;
|
||||
|
|
@ -128,6 +123,10 @@ class MLIDbStorage {
|
|||
.objectStore("configs")
|
||||
.add(DEFAULT_ML_SEARCH_CONFIG, ML_SEARCH_CONFIG_NAME);
|
||||
}
|
||||
if (oldVersion < 4) {
|
||||
db.deleteObjectStore("things");
|
||||
}
|
||||
|
||||
log.info(
|
||||
`Ml DB upgraded to version: ${newVersion} from version: ${oldVersion}`,
|
||||
);
|
||||
|
|
@ -299,21 +298,6 @@ class MLIDbStorage {
|
|||
log.info("updateFaces", Date.now() - startTime, "ms");
|
||||
}
|
||||
|
||||
public async getAllObjectsMap() {
|
||||
const startTime = Date.now();
|
||||
const db = await this.db;
|
||||
const allFiles = await db.getAll("files");
|
||||
const allObjectsMap = new Map<number, Array<RealWorldObject>>();
|
||||
allFiles.forEach(
|
||||
(mlFileData) =>
|
||||
mlFileData.objects &&
|
||||
allObjectsMap.set(mlFileData.fileId, mlFileData.objects),
|
||||
);
|
||||
log.info("allObjectsMap", Date.now() - startTime, "ms");
|
||||
|
||||
return allObjectsMap;
|
||||
}
|
||||
|
||||
public async getPerson(id: number) {
|
||||
const db = await this.db;
|
||||
return db.get("people", id);
|
||||
|
|
@ -334,20 +318,6 @@ class MLIDbStorage {
|
|||
return db.clear("people");
|
||||
}
|
||||
|
||||
public async getAllThings() {
|
||||
const db = await this.db;
|
||||
return db.getAll("things");
|
||||
}
|
||||
public async putThing(thing: Thing) {
|
||||
const db = await this.db;
|
||||
return db.put("things", thing);
|
||||
}
|
||||
|
||||
public async clearAllThings() {
|
||||
const db = await this.db;
|
||||
return db.clear("things");
|
||||
}
|
||||
|
||||
public async getIndexVersion(index: string) {
|
||||
const db = await this.db;
|
||||
return db.get("versions", index);
|
||||
|
|
|
|||
|
|
@ -57,14 +57,6 @@ function isSearchedFile(file: EnteFile, search: Search) {
|
|||
if (search?.person) {
|
||||
return search.person.files.indexOf(file.id) !== -1;
|
||||
}
|
||||
|
||||
if (search?.thing) {
|
||||
return search.thing.files.indexOf(file.id) !== -1;
|
||||
}
|
||||
|
||||
if (search?.text) {
|
||||
return search.text.files.indexOf(file.id) !== -1;
|
||||
}
|
||||
if (typeof search?.fileType !== "undefined") {
|
||||
return search.fileType === file.metadata.fileType;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { isDimensions, isValidNumber } from '../utils';
|
||||
import { IBoundingBox } from './BoundingBox';
|
||||
import { IDimensions } from './Dimensions';
|
||||
import { Point } from './Point';
|
||||
|
|
@ -172,4 +171,12 @@ export class Box<BoxType = any> implements IBoundingBox, IRect {
|
|||
bottom: this.bottom + (region.bottom * this.height)
|
||||
}).toSquare().round()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function isValidNumber(num: any) {
|
||||
return !!num && num !== Infinity && num !== -Infinity && !isNaN(num) || num === 0
|
||||
}
|
||||
|
||||
export function isDimensions(obj: any): boolean {
|
||||
return obj && obj.width && obj.height
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { isValidNumber } from '../utils';
|
||||
import { isValidNumber } from './Box';
|
||||
|
||||
export interface IDimensions {
|
||||
width: number
|
||||
|
|
@ -25,4 +25,4 @@ export class Dimensions implements IDimensions {
|
|||
public reverse(): Dimensions {
|
||||
return new Dimensions(1 / this.width, 1 / this.height)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
import * as tf from '@tensorflow/tfjs-core';
|
||||
|
||||
import { Point } from '../classes';
|
||||
import { Dimensions, IDimensions } from '../classes/Dimensions';
|
||||
|
||||
export function isTensor(tensor: any, dim: number) {
|
||||
return tensor instanceof tf.Tensor && tensor.shape.length === dim
|
||||
}
|
||||
|
||||
export function isTensor1D(tensor: any): tensor is tf.Tensor1D {
|
||||
return isTensor(tensor, 1)
|
||||
}
|
||||
|
||||
export function isTensor2D(tensor: any): tensor is tf.Tensor2D {
|
||||
return isTensor(tensor, 2)
|
||||
}
|
||||
|
||||
export function isTensor3D(tensor: any): tensor is tf.Tensor3D {
|
||||
return isTensor(tensor, 3)
|
||||
}
|
||||
|
||||
export function isTensor4D(tensor: any): tensor is tf.Tensor4D {
|
||||
return isTensor(tensor, 4)
|
||||
}
|
||||
|
||||
export function isFloat(num: number) {
|
||||
return num % 1 !== 0
|
||||
}
|
||||
|
||||
export function isEven(num: number) {
|
||||
return num % 2 === 0
|
||||
}
|
||||
|
||||
export function round(num: number, prec: number = 2) {
|
||||
const f = Math.pow(10, prec)
|
||||
return Math.floor(num * f) / f
|
||||
}
|
||||
|
||||
export function isDimensions(obj: any): boolean {
|
||||
return obj && obj.width && obj.height
|
||||
}
|
||||
|
||||
export function computeReshapedDimensions({ width, height }: IDimensions, inputSize: number) {
|
||||
const scale = inputSize / Math.max(height, width)
|
||||
return new Dimensions(Math.round(width * scale), Math.round(height * scale))
|
||||
}
|
||||
|
||||
export function getCenterPoint(pts: Point[]): Point {
|
||||
return pts.reduce((sum, pt) => sum.add(pt), new Point(0, 0))
|
||||
.div(new Point(pts.length, pts.length))
|
||||
}
|
||||
|
||||
export function range(num: number, start: number, step: number): number[] {
|
||||
return Array(num).fill(0).map((_, i) => start + (i * step))
|
||||
}
|
||||
|
||||
export function isValidNumber(num: any) {
|
||||
return !!num && num !== Infinity && num !== -Infinity && !isNaN(num) || num === 0
|
||||
}
|
||||
|
||||
export function isValidProbablitiy(num: any) {
|
||||
return isValidNumber(num) && 0 <= num && num <= 1.0
|
||||
}
|
||||
|
|
@ -954,52 +954,6 @@
|
|||
dependencies:
|
||||
tslib "^2.4.0"
|
||||
|
||||
"@tensorflow-models/coco-ssd@^2.2.2":
|
||||
version "2.2.3"
|
||||
resolved "https://registry.yarnpkg.com/@tensorflow-models/coco-ssd/-/coco-ssd-2.2.3.tgz#3825286569076d6788199c9cb89fb2fa31f7d2f2"
|
||||
integrity sha512-iCLGktG/XhHbP6h2FWxqCKMp/Px0lCp6MZU1fjNhjDHeaWEC9G7S7cZrnPXsfH+NewCM53YShlrHnknxU3SQig==
|
||||
|
||||
"@tensorflow/tfjs-backend-cpu@4.17.0", "@tensorflow/tfjs-backend-cpu@^4.10.0":
|
||||
version "4.17.0"
|
||||
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.17.0.tgz#b0c495de686cf700f2ae1f6d8bc2eb6f1964d250"
|
||||
integrity sha512-2VSCHnX9qhYTjw9HiVwTBSnRVlntKXeBlK7aSVsmZfHGwWE2faErTtO7bWmqNqw0U7gyznJbVAjlow/p+0RNGw==
|
||||
dependencies:
|
||||
"@types/seedrandom" "^2.4.28"
|
||||
seedrandom "^3.0.5"
|
||||
|
||||
"@tensorflow/tfjs-backend-webgl@^4.9.0":
|
||||
version "4.17.0"
|
||||
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.17.0.tgz#7d540a92343582d37d2cdf9509060598a19cd17a"
|
||||
integrity sha512-CC5GsGECCd7eYAUaKq0XJ48FjEZdgXZWPxgUYx4djvfUx5fQPp35hCSP9w/k463jllBMbjl2tKRg8u7Ia/LYzg==
|
||||
dependencies:
|
||||
"@tensorflow/tfjs-backend-cpu" "4.17.0"
|
||||
"@types/offscreencanvas" "~2019.3.0"
|
||||
"@types/seedrandom" "^2.4.28"
|
||||
seedrandom "^3.0.5"
|
||||
|
||||
"@tensorflow/tfjs-converter@^4.10.0":
|
||||
version "4.17.0"
|
||||
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-4.17.0.tgz#f4407bd53d5e300b05ed0b0f068506bc50c956b0"
|
||||
integrity sha512-qFxIjPfomCuTrYxsFjtKbi3QfdmTTCWo+RvqD64oCMS0sjp7sUDNhJyKDoLx6LZhXlwXpHIVDJctLMRMwet0Zw==
|
||||
|
||||
"@tensorflow/tfjs-core@^4.10.0":
|
||||
version "4.17.0"
|
||||
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-4.17.0.tgz#1ea128555a4d197aed417d70461fcbc7eaec635f"
|
||||
integrity sha512-v9Q5430EnRpyhWNd9LVgXadciKvxLiq+sTrLKRowh26BHyAsams4tZIgX3lFKjB7b90p+FYifVMcqLTTHgjGpQ==
|
||||
dependencies:
|
||||
"@types/long" "^4.0.1"
|
||||
"@types/offscreencanvas" "~2019.7.0"
|
||||
"@types/seedrandom" "^2.4.28"
|
||||
"@webgpu/types" "0.1.38"
|
||||
long "4.0.0"
|
||||
node-fetch "~2.6.1"
|
||||
seedrandom "^3.0.5"
|
||||
|
||||
"@tensorflow/tfjs-tflite@0.0.1-alpha.7":
|
||||
version "0.0.1-alpha.7"
|
||||
resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-tflite/-/tfjs-tflite-0.0.1-alpha.7.tgz#647c088689131fee424b7ae0bb9b7fdc74a61475"
|
||||
integrity sha512-aOmmEC/AHzfc/u1Q6ccY6Kr7CfNwjonqyTGVU1OqlQGDrH2IopcCjNSZdatJIB6J2RxlBs979JilCOUpK1LXng==
|
||||
|
||||
"@tokenizer/token@^0.3.0":
|
||||
version "0.3.0"
|
||||
resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.3.0.tgz#fe98a93fe789247e998c75e74e9c7c63217aa276"
|
||||
|
|
@ -1098,11 +1052,6 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.202.tgz#f09dbd2fb082d507178b2f2a5c7e74bd72ff98f8"
|
||||
integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ==
|
||||
|
||||
"@types/long@^4.0.1":
|
||||
version "4.0.2"
|
||||
resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a"
|
||||
integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==
|
||||
|
||||
"@types/node@*":
|
||||
version "20.11.20"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.20.tgz#f0a2aee575215149a62784210ad88b3a34843659"
|
||||
|
|
@ -1110,16 +1059,6 @@
|
|||
dependencies:
|
||||
undici-types "~5.26.4"
|
||||
|
||||
"@types/offscreencanvas@~2019.3.0":
|
||||
version "2019.3.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553"
|
||||
integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==
|
||||
|
||||
"@types/offscreencanvas@~2019.7.0":
|
||||
version "2019.7.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz#90267db13f64d6e9ccb5ae3eac92786a7c77a516"
|
||||
integrity sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==
|
||||
|
||||
"@types/parse-json@^4.0.0":
|
||||
version "4.0.2"
|
||||
resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.2.tgz#5950e50960793055845e956c427fc2b0d70c5239"
|
||||
|
|
@ -1195,11 +1134,6 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff"
|
||||
integrity sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==
|
||||
|
||||
"@types/seedrandom@^2.4.28":
|
||||
version "2.4.34"
|
||||
resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.34.tgz#c725cd0fc0442e2d3d0e5913af005686ffb7eb99"
|
||||
integrity sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A==
|
||||
|
||||
"@types/semver@^7.5.0":
|
||||
version "7.5.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.7.tgz#326f5fdda70d13580777bcaa1bc6fa772a5aef0e"
|
||||
|
|
@ -1363,11 +1297,6 @@
|
|||
"@types/babel__core" "^7.20.5"
|
||||
react-refresh "^0.14.0"
|
||||
|
||||
"@webgpu/types@0.1.38":
|
||||
version "0.1.38"
|
||||
resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.38.tgz#6fda4b410edc753d3213c648320ebcf319669020"
|
||||
integrity sha512-7LrhVKz2PRh+DD7+S+PVaFd5HxaWQvoMqBbsV9fNJO1pjUs1P8bM2vQVNfk+3URTqbuTI7gkXi0rfsN0IadoBA==
|
||||
|
||||
acorn-jsx@^5.3.2:
|
||||
version "5.3.2"
|
||||
resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937"
|
||||
|
|
@ -1630,11 +1559,6 @@ bip39@^3.0.4:
|
|||
dependencies:
|
||||
"@noble/hashes" "^1.2.0"
|
||||
|
||||
blazeface-back@^0.0.9:
|
||||
version "0.0.9"
|
||||
resolved "https://registry.yarnpkg.com/blazeface-back/-/blazeface-back-0.0.9.tgz#a8a26a0022950eb21136693f2fca3c52315ad2a4"
|
||||
integrity sha512-t0i5V117j074d7d7mlLaRq9n/bYchXcSEgpWVbGGloV68A6Jn22t4SNoEC3t+MOsU8H+eXoDv2/6+JsqActM1g==
|
||||
|
||||
brace-expansion@^1.1.7:
|
||||
version "1.1.11"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
|
||||
|
|
@ -3445,11 +3369,6 @@ lodash@^4.17.21:
|
|||
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
|
||||
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
|
||||
|
||||
long@4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
|
||||
integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==
|
||||
|
||||
loose-envify@^1.1.0, loose-envify@^1.4.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
|
||||
|
|
@ -3621,13 +3540,6 @@ node-fetch@^2.6.1:
|
|||
dependencies:
|
||||
whatwg-url "^5.0.0"
|
||||
|
||||
node-fetch@~2.6.1:
|
||||
version "2.6.13"
|
||||
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.13.tgz#a20acbbec73c2e09f9007de5cda17104122e0010"
|
||||
integrity sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==
|
||||
dependencies:
|
||||
whatwg-url "^5.0.0"
|
||||
|
||||
node-releases@^2.0.14:
|
||||
version "2.0.14"
|
||||
resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b"
|
||||
|
|
@ -4251,11 +4163,6 @@ scheduler@^0.23.0:
|
|||
dependencies:
|
||||
loose-envify "^1.1.0"
|
||||
|
||||
seedrandom@^3.0.5:
|
||||
version "3.0.5"
|
||||
resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7"
|
||||
integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==
|
||||
|
||||
semver@^6.3.1:
|
||||
version "6.3.1"
|
||||
resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue