diff --git a/web/apps/photos/src/components/MachineLearning/ObjectList.tsx b/web/apps/photos/src/components/MachineLearning/ObjectList.tsx deleted file mode 100644 index f22ccc06f..000000000 --- a/web/apps/photos/src/components/MachineLearning/ObjectList.tsx +++ /dev/null @@ -1,51 +0,0 @@ -import Box from "@mui/material/Box"; -import { Chip } from "components/Chip"; -import { Legend } from "components/PhotoViewer/styledComponents/Legend"; -import { t } from "i18next"; -import { useEffect, useState } from "react"; -import { EnteFile } from "types/file"; -import mlIDbStorage from "utils/storage/mlIDbStorage"; - -export function ObjectLabelList(props: { - file: EnteFile; - updateMLDataIndex: number; -}) { - const [objects, setObjects] = useState>([]); - useEffect(() => { - let didCancel = false; - const main = async () => { - const objects = await mlIDbStorage.getAllObjectsMap(); - const uniqueObjectNames = [ - ...new Set( - (objects.get(props.file.id) ?? []).map( - (object) => object.detection.class, - ), - ), - ]; - !didCancel && setObjects(uniqueObjectNames); - }; - main(); - return () => { - didCancel = true; - }; - }, [props.file, props.updateMLDataIndex]); - - if (objects.length === 0) return <>; - - return ( -
- {t("OBJECTS")} - - {objects.map((object) => ( - {object} - ))} - -
- ); -} diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 0b4054e27..24de200c0 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -10,7 +10,6 @@ import TextSnippetOutlined from "@mui/icons-material/TextSnippetOutlined"; import { Box, DialogProps, Link, Stack, styled } from "@mui/material"; import { Chip } from "components/Chip"; import { EnteDrawer } from "components/EnteDrawer"; -import { ObjectLabelList } from "components/MachineLearning/ObjectList"; import { PhotoPeopleList, UnidentifiedFaces, @@ -344,10 +343,6 @@ export function FileInfo({ file={file} updateMLDataIndex={updateMLDataIndex} /> - )} diff --git a/web/apps/photos/src/services/machineLearning/objectService.ts b/web/apps/photos/src/services/machineLearning/objectService.ts deleted file mode 100644 index 97d7895a5..000000000 --- a/web/apps/photos/src/services/machineLearning/objectService.ts +++ /dev/null @@ -1,146 +0,0 @@ -import log from "@/next/log"; -import { - DetectedObject, - MLSyncContext, - MLSyncFileContext, - Thing, -} from "types/machineLearning"; -import { - getAllObjectsFromMap, - getObjectId, - isDifferentOrOld, -} from "utils/machineLearning"; -import mlIDbStorage from "utils/storage/mlIDbStorage"; -import ReaderService from "./readerService"; - -class ObjectService { - async syncFileObjectDetections( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - ) { - const startTime = Date.now(); - const { oldMlFile, newMlFile } = fileContext; - if ( - !isDifferentOrOld( - oldMlFile?.objectDetectionMethod, - syncContext.objectDetectionService.method, - ) && - !isDifferentOrOld( - oldMlFile?.sceneDetectionMethod, - syncContext.sceneDetectionService.method, - ) && - oldMlFile?.imageSource === syncContext.config.imageSource - ) { - newMlFile.objects = oldMlFile?.objects; - newMlFile.imageSource = oldMlFile.imageSource; - newMlFile.imageDimensions = oldMlFile.imageDimensions; - newMlFile.objectDetectionMethod = oldMlFile.objectDetectionMethod; - newMlFile.sceneDetectionMethod = oldMlFile.sceneDetectionMethod; - return; - } - - newMlFile.objectDetectionMethod = - syncContext.objectDetectionService.method; - newMlFile.sceneDetectionMethod = - syncContext.sceneDetectionService.method; - - fileContext.newDetection = true; - const imageBitmap = await ReaderService.getImageBitmap( - syncContext, - fileContext, - ); - const objectDetections = - await syncContext.objectDetectionService.detectObjects( - imageBitmap, - syncContext.config.objectDetection.maxNumBoxes, - syncContext.config.objectDetection.minScore, - ); - objectDetections.push( - ...(await syncContext.sceneDetectionService.detectScenes( - imageBitmap, - syncContext.config.sceneDetection.minScore, - )), - ); - // log.info('3 TF Memory stats: ',JSON.stringify(tf.memory())); - // TODO: reenable faces filtering based on width - const detectedObjects = objectDetections?.map((detection) => { - return { - fileID: fileContext.enteFile.id, - detection, - } as DetectedObject; - }); - newMlFile.objects = detectedObjects?.map((detectedObject) => ({ - ...detectedObject, - id: getObjectId(detectedObject, newMlFile.imageDimensions), - className: detectedObject.detection.class, - })); - // ?.filter((f) => - // f.box.width > syncContext.config.faceDetection.minFaceSize - // ); - log.info( - `object detection time taken ${fileContext.enteFile.id}`, - Date.now() - startTime, - "ms", - ); - - log.info("[MLService] Detected Objects: ", newMlFile.objects?.length); - } - - async getAllSyncedObjectsMap(syncContext: MLSyncContext) { - if (syncContext.allSyncedObjectsMap) { - return syncContext.allSyncedObjectsMap; - } - - syncContext.allSyncedObjectsMap = await mlIDbStorage.getAllObjectsMap(); - return syncContext.allSyncedObjectsMap; - } - - public async clusterThings(syncContext: MLSyncContext): Promise { - const allObjectsMap = await this.getAllSyncedObjectsMap(syncContext); - const allObjects = getAllObjectsFromMap(allObjectsMap); - const objectClusters = new Map(); - allObjects.map((object) => { - if (!objectClusters.has(object.detection.class)) { - objectClusters.set(object.detection.class, []); - } - const objectsInCluster = objectClusters.get(object.detection.class); - objectsInCluster.push(object.fileID); - }); - return [...objectClusters.entries()].map(([className, files], id) => ({ - id, - name: className, - files, - })); - } - - async syncThingsIndex(syncContext: MLSyncContext) { - const filesVersion = await mlIDbStorage.getIndexVersion("files"); - log.info("things", await mlIDbStorage.getIndexVersion("things")); - if (filesVersion <= (await mlIDbStorage.getIndexVersion("things"))) { - log.info( - "[MLService] Skipping people index as already synced to latest version", - ); - return; - } - - const things = await this.clusterThings(syncContext); - - if (!things || things.length < 1) { - return; - } - - await mlIDbStorage.clearAllThings(); - - for (const thing of things) { - await mlIDbStorage.putThing(thing); - } - - await mlIDbStorage.setIndexVersion("things", filesVersion); - } - - async getAllThings() { - return await mlIDbStorage.getAllThings(); - } -} - -export default new ObjectService(); diff --git a/web/apps/photos/src/types/machineLearning/index.ts b/web/apps/photos/src/types/machineLearning/index.ts index 712ffa73f..ddceb188b 100644 --- a/web/apps/photos/src/types/machineLearning/index.ts +++ b/web/apps/photos/src/types/machineLearning/index.ts @@ -161,22 +161,6 @@ export interface ObjectDetection { score: number; } -export interface DetectedObject { - fileID: number; - detection: ObjectDetection; -} - -export interface RealWorldObject extends DetectedObject { - id: string; - className: string; -} - -export interface Thing { - id: number; - name: string; - files: Array; -} - export interface WordGroup { word: string; files: Array; @@ -185,7 +169,6 @@ export interface WordGroup { export interface MlFileData { fileId: number; faces?: Face[]; - objects?: RealWorldObject[]; imageSource?: ImageType; imageDimensions?: Dimensions; faceDetectionMethod?: Versioned; diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index b35324000..6603dde69 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -15,13 +15,11 @@ import { Dimensions } from "types/image"; import { AlignedFace, DetectedFace, - DetectedObject, Face, FaceAlignment, FaceImageBlob, MlFileData, Person, - RealWorldObject, Versioned, } from "types/machineLearning"; import { getRenderableImage } from "utils/file"; @@ -196,12 +194,6 @@ export function getAllFacesFromMap(allFacesMap: Map>) { return allFaces; } -export function getAllObjectsFromMap( - allObjectsMap: Map>, -) { - return [...allObjectsMap.values()].flat(); -} - export async function getLocalFile(fileId: number) { const localFiles = await getLocalFiles(); return localFiles.find((f) => f.id === fileId); @@ -312,26 +304,6 @@ export function getFaceId(detectedFace: DetectedFace, imageDims: Dimensions) { return faceID; } -export function getObjectId( - detectedObject: DetectedObject, - imageDims: Dimensions, -) { - const imgDimPoint = new Point(imageDims.width, imageDims.height); - const objectCenterPoint = new Point( - detectedObject.detection.bbox[2] / 2, - detectedObject.detection.bbox[3] / 2, - ); - const gridPt = objectCenterPoint - .mul(new Point(100, 100)) - .div(imgDimPoint) - .floor() - .bound(0, 99); - const gridPaddedX = leftFillNum(gridPt.x, 2, 0); - const gridPaddedY = leftFillNum(gridPt.y, 2, 0); - - return `${detectedObject.fileID}-${gridPaddedX}-${gridPaddedY}`; -} - export async function getTFImage(blob): Promise { const imageBitmap = await createImageBitmap(blob); const tfImage = tf.browser.fromPixels(imageBitmap); diff --git a/web/apps/photos/src/utils/storage/mlIDbStorage.ts b/web/apps/photos/src/utils/storage/mlIDbStorage.ts index 96b2cd905..d7e24cbe8 100644 --- a/web/apps/photos/src/utils/storage/mlIDbStorage.ts +++ b/web/apps/photos/src/utils/storage/mlIDbStorage.ts @@ -15,14 +15,7 @@ import { openDB, } from "idb"; import isElectron from "is-electron"; -import { - Face, - MLLibraryData, - MlFileData, - Person, - RealWorldObject, - Thing, -} from "types/machineLearning"; +import { Face, MLLibraryData, MlFileData, Person } from "types/machineLearning"; import { IndexStatus } from "types/machineLearning/ui"; interface Config {} @@ -42,9 +35,11 @@ interface MLDb extends DBSchema { key: number; value: Person; }; + // Unused, we only retain this is the schema so that we can delete it during + // migration. things: { key: number; - value: Thing; + value: unknown; }; versions: { key: string; @@ -72,7 +67,7 @@ class MLIDbStorage { } private openDB(): Promise> { - return openDB(MLDATA_DB_NAME, 3, { + return openDB(MLDATA_DB_NAME, 4, { terminated: async () => { log.error("ML Indexed DB terminated"); this._db = undefined; @@ -128,6 +123,10 @@ class MLIDbStorage { .objectStore("configs") .add(DEFAULT_ML_SEARCH_CONFIG, ML_SEARCH_CONFIG_NAME); } + if (oldVersion < 4) { + db.deleteObjectStore("things"); + } + log.info( `Ml DB upgraded to version: ${newVersion} from version: ${oldVersion}`, ); @@ -299,21 +298,6 @@ class MLIDbStorage { log.info("updateFaces", Date.now() - startTime, "ms"); } - public async getAllObjectsMap() { - const startTime = Date.now(); - const db = await this.db; - const allFiles = await db.getAll("files"); - const allObjectsMap = new Map>(); - allFiles.forEach( - (mlFileData) => - mlFileData.objects && - allObjectsMap.set(mlFileData.fileId, mlFileData.objects), - ); - log.info("allObjectsMap", Date.now() - startTime, "ms"); - - return allObjectsMap; - } - public async getPerson(id: number) { const db = await this.db; return db.get("people", id); @@ -334,20 +318,6 @@ class MLIDbStorage { return db.clear("people"); } - public async getAllThings() { - const db = await this.db; - return db.getAll("things"); - } - public async putThing(thing: Thing) { - const db = await this.db; - return db.put("things", thing); - } - - public async clearAllThings() { - const db = await this.db; - return db.clear("things"); - } - public async getIndexVersion(index: string) { const db = await this.db; return db.get("versions", index);