diff --git a/web/apps/photos/src/services/face/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts index 8c9d09498..cc4d7017c 100644 --- a/web/apps/photos/src/services/face/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -19,7 +19,7 @@ export class DedicatedMLWorker implements MachineLearningWorker { enteFile: EnteFile, localFile: globalThis.File, ) { - return mlService.syncLocalFile(token, userID, enteFile, localFile); + mlService.syncLocalFile(token, userID, enteFile, localFile); } public async sync(token: string, userID: number) { diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index f7a88d9c6..9d2c4536f 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -197,7 +197,6 @@ export interface MLSearchConfig { export interface MLSyncContext { token: string; userID: number; - shouldUpdateMLVersion: boolean; faceDetectionService: FaceDetectionService; faceCropService: FaceCropService; @@ -281,7 +280,7 @@ export interface MachineLearningWorker { userID: number, enteFile: EnteFile, localFile: globalThis.File, - ): Promise; + ); sync(token: string, userID: number): Promise; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 8b521add9..f5e83b8f5 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -8,7 +8,6 @@ import { MLSyncContext, MLSyncFileContext, type FaceAlignment, - type Versioned, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; import { clusterFaces } from "../face/cluster"; @@ -24,36 +23,12 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - !isDifferentOrOld( - oldMlFile?.faceDetectionMethod, - syncContext.faceDetectionService.method, - ) && - oldMlFile?.imageSource === "Original" - ) { - newMlFile.faces = oldMlFile?.faces?.map((existingFace) => ({ - id: existingFace.id, - fileId: existingFace.fileId, - detection: existingFace.detection, - })); - - newMlFile.imageSource = oldMlFile.imageSource; - newMlFile.imageDimensions = oldMlFile.imageDimensions; - newMlFile.faceDetectionMethod = oldMlFile.faceDetectionMethod; - return; - } - + const { newMlFile } = fileContext; newMlFile.faceDetectionMethod = syncContext.faceDetectionService.method; fileContext.newDetection = true; const imageBitmap = await fetchImageBitmapForContext(fileContext); - const timerId = `faceDetection-${fileContext.enteFile.id}`; - console.time(timerId); const faceDetections = await syncContext.faceDetectionService.detectFaces(imageBitmap); - console.timeEnd(timerId); - console.log("faceDetections: ", faceDetections?.length); - // TODO: reenable faces filtering based on width const detectedFaces = faceDetections?.map((detection) => { return { @@ -75,23 +50,7 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - // !syncContext.config.faceCrop.enabled || - !fileContext.newDetection && - !isDifferentOrOld( - oldMlFile?.faceCropMethod, - syncContext.faceCropService.method, - ) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - for (const [index, face] of newMlFile.faces.entries()) { - face.crop = oldMlFile.faces[index].crop; - } - newMlFile.faceCropMethod = oldMlFile.faceCropMethod; - return; - } - + const { newMlFile } = fileContext; const imageBitmap = await fetchImageBitmapForContext(fileContext); newMlFile.faceCropMethod = syncContext.faceCropService.method; @@ -104,24 +63,7 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ): Promise { - const { oldMlFile, newMlFile } = fileContext; - // TODO-ML(MR): - const method = { - value: "ArcFace", - version: 1, - }; - if ( - !fileContext.newDetection && - !isDifferentOrOld(oldMlFile?.faceAlignmentMethod, method) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - for (const [index, face] of newMlFile.faces.entries()) { - face.alignment = oldMlFile.faces[index].alignment; - } - newMlFile.faceAlignmentMethod = oldMlFile.faceAlignmentMethod; - return; - } - + const { newMlFile } = fileContext; newMlFile.faceAlignmentMethod = { value: "ArcFace", version: 1, @@ -159,22 +101,7 @@ class FaceService { fileContext: MLSyncFileContext, alignedFacesInput: Float32Array, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - !fileContext.newAlignment && - !isDifferentOrOld( - oldMlFile?.faceEmbeddingMethod, - syncContext.faceEmbeddingService.method, - ) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - for (const [index, face] of newMlFile.faces.entries()) { - face.embedding = oldMlFile.faces[index].embedding; - } - newMlFile.faceEmbeddingMethod = oldMlFile.faceEmbeddingMethod; - return; - } - + const { newMlFile } = fileContext; newMlFile.faceEmbeddingMethod = syncContext.faceEmbeddingService.method; // TODO: when not storing face crops, image will be needed to extract faces // fileContext.imageBitmap || @@ -193,17 +120,7 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - !fileContext.newAlignment && - !isDifferentOrOld( - oldMlFile?.faceEmbeddingMethod, - syncContext.faceEmbeddingService.method, - ) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - return; - } + const { newMlFile } = fileContext; for (let i = 0; i < newMlFile.faces.length; i++) { const face = newMlFile.faces[i]; if (face.detection.box.x + face.detection.box.width < 2) continue; // Skip if somehow already relative @@ -298,39 +215,6 @@ class FaceService { export default new FaceService(); -export function areFaceIdsSame(ofFaces: Array, toFaces: Array) { - if ( - (ofFaces === null || ofFaces === undefined) && - (toFaces === null || toFaces === undefined) - ) { - return true; - } - return primitiveArrayEquals( - ofFaces?.map((f) => f.id), - toFaces?.map((f) => f.id), - ); -} - -function primitiveArrayEquals(a, b) { - return ( - Array.isArray(a) && - Array.isArray(b) && - a.length === b.length && - a.every((val, index) => val === b[index]) - ); -} - -export function isDifferentOrOld( - method: Versioned, - thanMethod: Versioned, -) { - return ( - !method || - method.value !== thanMethod.value || - method.version < thanMethod.version - ); -} - async function extractFaceImagesToFloat32( faceAlignments: Array, faceSize: number, diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 26d9e69dd..058f2f001 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -157,7 +157,6 @@ export class MLFactory { export class LocalMLSyncContext implements MLSyncContext { public token: string; public userID: number; - public shouldUpdateMLVersion: boolean; public faceDetectionService: FaceDetectionService; public faceCropService: FaceCropService; @@ -184,15 +183,9 @@ export class LocalMLSyncContext implements MLSyncContext { >; private enteWorkers: Array; - constructor( - token: string, - userID: number, - shouldUpdateMLVersion: boolean = true, - concurrency?: number, - ) { + constructor(token: string, userID: number, concurrency?: number) { this.token = token; this.userID = userID; - this.shouldUpdateMLVersion = shouldUpdateMLVersion; this.faceDetectionService = MLFactory.getFaceDetectionService("YoloFace"); @@ -424,7 +417,7 @@ class MachineLearningService { // TODO-ML(MR): Keep as promise for now. this.syncContext = new Promise((resolve) => { - resolve(new LocalMLSyncContext(token, userID, true)); + resolve(new LocalMLSyncContext(token, userID)); }); } else { log.info("reusing existing syncContext"); @@ -433,11 +426,12 @@ class MachineLearningService { } private async getLocalSyncContext(token: string, userID: number) { + // TODO-ML(MR): This is updating the file ML version. verify. if (!this.localSyncContext) { log.info("Creating localSyncContext"); // TODO-ML(MR): this.localSyncContext = new Promise((resolve) => { - resolve(new LocalMLSyncContext(token, userID, false)); + resolve(new LocalMLSyncContext(token, userID)); }); } else { log.info("reusing existing localSyncContext"); @@ -459,11 +453,11 @@ class MachineLearningService { userID: number, enteFile: EnteFile, localFile?: globalThis.File, - ): Promise { + ) { const syncContext = await this.getLocalSyncContext(token, userID); try { - const mlFileData = await this.syncFileWithErrorHandler( + await this.syncFileWithErrorHandler( syncContext, enteFile, localFile, @@ -473,10 +467,8 @@ class MachineLearningService { await this.closeLocalSyncContext(); } // await syncContext.dispose(); - return mlFileData; } catch (e) { console.error("Error while syncing local file: ", enteFile.id, e); - return e; } } @@ -484,7 +476,7 @@ class MachineLearningService { syncContext: MLSyncContext, enteFile: EnteFile, localFile?: globalThis.File, - ): Promise { + ) { try { console.log( `Indexing ${enteFile.title ?? ""} ${enteFile.id}`, @@ -533,22 +525,13 @@ class MachineLearningService { ) { console.log("Syncing for file" + enteFile.title); const fileContext: MLSyncFileContext = { enteFile, localFile }; - const oldMlFile = - (fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ?? - this.newMlData(enteFile.id); - if ( - fileContext.oldMlFile?.mlVersion === defaultMLVersion - // TODO: reset mlversion of all files when user changes image source - ) { - return fileContext.oldMlFile; + const oldMlFile = await this.getMLFileData(enteFile.id); + if (oldMlFile) { + return oldMlFile; } - const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id)); - if (syncContext.shouldUpdateMLVersion) { - newMlFile.mlVersion = defaultMLVersion; - } else if (fileContext.oldMlFile?.mlVersion) { - newMlFile.mlVersion = fileContext.oldMlFile.mlVersion; - } + const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id)); + newMlFile.mlVersion = defaultMLVersion; try { await fetchImageBitmapForContext(fileContext); @@ -628,6 +611,7 @@ class MachineLearningService { public async syncIndex(syncContext: MLSyncContext) { await this.getMLLibraryData(syncContext); + // TODO-ML(MR): Ensure this doesn't run until fixed. await syncPeopleIndex(syncContext); await this.persistMLLibraryData(syncContext); diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 52fb43f9c..8f58965f1 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -5,8 +5,8 @@ import { eventBus, Events } from "@ente/shared/events"; import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; import debounce from "debounce"; import PQueue from "p-queue"; -import mlIDbStorage from "services/face/db"; import { createFaceComlinkWorker } from "services/face"; +import mlIDbStorage from "services/face/db"; import type { DedicatedMLWorker } from "services/face/face.worker"; import { MLSyncResult } from "services/face/types"; import { EnteFile } from "types/file"; @@ -232,19 +232,13 @@ class MLWorkManager { } public async syncLocalFile(enteFile: EnteFile, localFile: globalThis.File) { - const result = await this.liveSyncQueue.add(async () => { + await this.liveSyncQueue.add(async () => { this.stopSyncJob(); const token = getToken(); const userID = getUserID(); const mlWorker = await this.getLiveSyncWorker(); return mlWorker.syncLocalFile(token, userID, enteFile, localFile); }); - - if (result instanceof Error) { - // TODO: redirect/refresh to gallery in case of session_expired - // may not be required as uploader should anyways take care of this - console.error("Error while syncing local file: ", result); - } } // Sync Job