diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx index 6ebc0d942..9b5d59eac 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx @@ -5,7 +5,7 @@ import { t } from "i18next"; import { AppContext } from "pages/_app"; import { useContext } from "react"; import { components } from "react-select"; -import { IndexStatus } from "types/machineLearning/ui"; +import { IndexStatus } from "utils/storage/mlIDbStorage"; import { Suggestion, SuggestionType } from "types/search"; const { Menu } = components; diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx index 3f737b3e0..3b995b849 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx @@ -17,7 +17,7 @@ import { import { Collection } from "types/collection"; import { LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; -import { Person } from "types/machineLearning"; +import { Person } from "services/ml/types"; import { ClipSearchScores, DateValue, diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index 4691d4b65..ea317fbec 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -4,7 +4,7 @@ import { Legend } from "components/PhotoViewer/styledComponents/Legend"; import { t } from "i18next"; import React, { useEffect, useState } from "react"; import { EnteFile } from "types/file"; -import { Face, Person } from "types/machineLearning"; +import { Face, Person } from "services/ml/types"; import { getPeopleList, getUnidentifiedFaces } from "utils/machineLearning"; const FaceChipContainer = styled("div")` diff --git a/web/apps/photos/src/constants/mlConfig.ts b/web/apps/photos/src/constants/mlConfig.ts index 929594e1c..d8aaeb410 100644 --- a/web/apps/photos/src/constants/mlConfig.ts +++ b/web/apps/photos/src/constants/mlConfig.ts @@ -1,5 +1,5 @@ import { JobConfig } from "types/common/job"; -import { MLSearchConfig, MLSyncConfig } from "types/machineLearning"; +import { MLSearchConfig, MLSyncConfig } from "services/ml/types"; export const DEFAULT_ML_SYNC_JOB_CONFIG: JobConfig = { intervalSec: 5, diff --git a/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts b/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts index 99063b3f2..1d7deac5e 100644 --- a/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts +++ b/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts @@ -4,7 +4,7 @@ import { FaceAlignmentService, FaceDetection, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; import { getArcfaceAlignment } from "utils/machineLearning/faceAlign"; class ArcfaceAlignmentService implements FaceAlignmentService { diff --git a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts index cb6ccd029..f5e1e835a 100644 --- a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts +++ b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts @@ -5,7 +5,7 @@ import { FaceCropService, FaceDetection, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; import { getArcfaceAlignment } from "utils/machineLearning/faceAlign"; import { getFaceCrop } from "utils/machineLearning/faceCrop"; diff --git a/web/apps/photos/src/services/machineLearning/clusteringService.ts b/web/apps/photos/src/services/machineLearning/clusteringService.ts index 03931b63b..32c25f698 100644 --- a/web/apps/photos/src/services/machineLearning/clusteringService.ts +++ b/web/apps/photos/src/services/machineLearning/clusteringService.ts @@ -8,7 +8,7 @@ import { ClusteringResults, HdbscanResults, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; class ClusteringService { private dbscan: DBSCAN; diff --git a/web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts index 33298eef3..57d181de4 100644 --- a/web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts +++ b/web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts @@ -6,7 +6,7 @@ import { ClusteringService, HdbscanResults, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; class DbscanClusteringService implements ClusteringService { public method: Versioned; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 1dedadf15..6bb045ae2 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -5,7 +5,7 @@ import { Face, MLSyncContext, MLSyncFileContext, -} from "types/machineLearning"; +} from "services/ml/types"; import { imageBitmapToBlob } from "utils/image"; import { areFaceIdsSame, diff --git a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts index 21e211825..0671b0bde 100644 --- a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts +++ b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts @@ -6,7 +6,7 @@ import { ClusteringService, HdbscanResults, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; class HdbscanClusteringService implements ClusteringService { public method: Versioned; diff --git a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts index 3357e21cc..f1d7bf500 100644 --- a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts +++ b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts @@ -3,7 +3,7 @@ import { BlurDetectionService, Face, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; import { createGrayscaleIntMatrixFromNormalized2List } from "utils/image"; import { mobileFaceNetFaceSize } from "./mobileFaceNetEmbeddingService"; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningFactory.ts b/web/apps/photos/src/services/machineLearning/machineLearningFactory.ts index 991ae6808..999bae8fe 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningFactory.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningFactory.ts @@ -4,7 +4,6 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { getDedicatedCryptoWorker } from "@ente/shared/crypto"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import PQueue from "p-queue"; -import { EnteFile } from "types/file"; import { BlurDetectionMethod, BlurDetectionService, @@ -22,7 +21,8 @@ import { MLLibraryData, MLSyncConfig, MLSyncContext, -} from "types/machineLearning"; +} from "services/ml/types"; +import { EnteFile } from "types/file"; import { logQueueStats } from "utils/machineLearning"; import arcfaceAlignmentService from "./arcfaceAlignmentService"; import arcfaceCropService from "./arcfaceCropService"; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 03a3b7e2c..f2354f7b1 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -6,13 +6,13 @@ import { MAX_ML_SYNC_ERROR_COUNT } from "constants/mlConfig"; import downloadManager from "services/download"; import { putEmbedding } from "services/embeddingService"; import { getLocalFiles } from "services/fileService"; -import { EnteFile } from "types/file"; import { MLSyncContext, MLSyncFileContext, MLSyncResult, MlFileData, -} from "types/machineLearning"; +} from "services/ml/types"; +import { EnteFile } from "types/file"; import { getMLSyncConfig } from "utils/machineLearning/config"; import { LocalFileMlDataToServerFileMl } from "utils/machineLearning/mldataMappers"; import mlIDbStorage from "utils/storage/mlIDbStorage"; diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index d1c5e9db5..c75d4b14a 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -5,9 +5,9 @@ import { eventBus, Events } from "@ente/shared/events"; import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; import debounce from "debounce"; import PQueue from "p-queue"; +import { MLSyncResult } from "services/ml/types"; import { JobResult } from "types/common/job"; import { EnteFile } from "types/file"; -import { MLSyncResult } from "types/machineLearning"; import { getDedicatedMLWorker } from "utils/comlink/ComlinkMLWorker"; import { SimpleJob } from "utils/common/job"; import { logQueueStats } from "utils/machineLearning"; diff --git a/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts b/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts index 818b8a5d1..1b2205801 100644 --- a/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts +++ b/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts @@ -4,7 +4,7 @@ import { FaceEmbeddingMethod, FaceEmbeddingService, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; export const mobileFaceNetFaceSize = 112; diff --git a/web/apps/photos/src/services/machineLearning/peopleService.ts b/web/apps/photos/src/services/machineLearning/peopleService.ts index ad7d7bcec..f4ed5104b 100644 --- a/web/apps/photos/src/services/machineLearning/peopleService.ts +++ b/web/apps/photos/src/services/machineLearning/peopleService.ts @@ -1,5 +1,5 @@ import log from "@/next/log"; -import { Face, MLSyncContext, Person } from "types/machineLearning"; +import { Face, MLSyncContext, Person } from "services/ml/types"; import { findFirstIfSorted, getAllFacesFromMap, diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/machineLearning/readerService.ts index 62aebdbd1..44d0738f5 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/machineLearning/readerService.ts @@ -1,6 +1,6 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; -import { MLSyncContext, MLSyncFileContext } from "types/machineLearning"; +import { MLSyncContext, MLSyncFileContext } from "services/ml/types"; import { getLocalFileImageBitmap, getOriginalImageBitmap, diff --git a/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts b/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts index 4fa840749..b9a2ed1b6 100644 --- a/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts +++ b/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts @@ -1,5 +1,11 @@ import { workerBridge } from "@/next/worker/worker-bridge"; import { euclidean } from "hdbscan"; +import { + FaceDetection, + FaceDetectionMethod, + FaceDetectionService, + Versioned, +} from "services/ml/types"; import { Matrix, applyToPoint, @@ -8,12 +14,6 @@ import { translate, } from "transformation-matrix"; import { Dimensions } from "types/image"; -import { - FaceDetection, - FaceDetectionMethod, - FaceDetectionService, - Versioned, -} from "types/machineLearning"; import { clamp, getPixelBilinear, diff --git a/web/apps/photos/src/types/machineLearning/index.ts b/web/apps/photos/src/services/ml/types.ts similarity index 89% rename from web/apps/photos/src/types/machineLearning/index.ts rename to web/apps/photos/src/services/ml/types.ts index 2c3961cdf..1c95717f5 100644 --- a/web/apps/photos/src/types/machineLearning/index.ts +++ b/web/apps/photos/src/services/ml/types.ts @@ -329,3 +329,46 @@ export interface MachineLearningWorker { close(): void; } + +export interface ClipEmbedding { + embedding: Float32Array; + model: "ggml-clip" | "onnx-clip"; +} + +/// [`x`] and [y] are the coordinates of the top left corner of the box, so the minimim values +/// [width] and [height] are the width and height of the box. +/// All values are in absolute pixels relative to the original image size. +export interface CenterBox { + x: number; + y: number; + height: number; + width: number; +} + +export interface DetectionPoint { + x: number; + y: number; +} + +export interface Detection { + box: CenterBox; + landmarks: DetectionPoint[]; +} + +export interface FileMLFace { + id: string; + confidence: number; + blur: number; + embedding: Float32Array; + detection: Detection; +} + +export interface FileML { + fileID: number; + clip?: ClipEmbedding; + faces: Face[]; + height: number; + width: number; + version: number; + error?: string; +} diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 96c574b9d..7c00dec72 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -2,10 +2,10 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import * as chrono from "chrono-node"; import { t } from "i18next"; +import { Person } from "services/ml/types"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; -import { Person } from "types/machineLearning"; import { ClipSearchScores, DateValue, diff --git a/web/apps/photos/src/types/machineLearning/data/clip.ts b/web/apps/photos/src/types/machineLearning/data/clip.ts deleted file mode 100644 index 0181e89e5..000000000 --- a/web/apps/photos/src/types/machineLearning/data/clip.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface ClipEmbedding { - embedding: Float32Array; - model: "ggml-clip" | "onnx-clip"; -} diff --git a/web/apps/photos/src/types/machineLearning/data/face.ts b/web/apps/photos/src/types/machineLearning/data/face.ts deleted file mode 100644 index cac391994..000000000 --- a/web/apps/photos/src/types/machineLearning/data/face.ts +++ /dev/null @@ -1,27 +0,0 @@ -/// [`x`] and [y] are the coordinates of the top left corner of the box, so the minimim values -/// [width] and [height] are the width and height of the box. -/// All values are in absolute pixels relative to the original image size. -export interface CenterBox { - x: number; - y: number; - height: number; - width: number; -} - -export interface Point { - x: number; - y: number; -} - -export interface Detection { - box: CenterBox; - landmarks: Point[]; -} - -export interface Face { - id: string; - confidence: number; - blur: number; - embedding: Float32Array; - detection: Detection; -} diff --git a/web/apps/photos/src/types/machineLearning/data/fileML.ts b/web/apps/photos/src/types/machineLearning/data/fileML.ts deleted file mode 100644 index 7835450e7..000000000 --- a/web/apps/photos/src/types/machineLearning/data/fileML.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { ClipEmbedding } from "./clip"; -import { Face } from "./face"; - -export interface FileML { - fileID: number; - clip?: ClipEmbedding; - faces: Face[]; - height: number; - width: number; - version: number; - error?: string; -} diff --git a/web/apps/photos/src/types/machineLearning/ui.ts b/web/apps/photos/src/types/machineLearning/ui.ts index cd9f63f18..e69de29bb 100644 --- a/web/apps/photos/src/types/machineLearning/ui.ts +++ b/web/apps/photos/src/types/machineLearning/ui.ts @@ -1,7 +0,0 @@ -export interface IndexStatus { - outOfSyncFilesExists: boolean; - nSyncedFiles: number; - nTotalFiles: number; - localFilesSynced: boolean; - peopleIndexSynced: boolean; -} diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index cf50f4a06..7cff13298 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,9 +1,9 @@ import { FILE_TYPE } from "@/media/file-type"; import { City } from "services/locationSearchService"; +import { Person } from "services/ml/types"; import { LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; -import { Person } from "types/machineLearning"; -import { IndexStatus } from "types/machineLearning/ui"; +import { IndexStatus } from "utils/storage/mlIDbStorage"; export enum SuggestionType { DATE = "DATE", diff --git a/web/apps/photos/src/utils/image/index.ts b/web/apps/photos/src/utils/image/index.ts index e4884716c..38fe248c8 100644 --- a/web/apps/photos/src/utils/image/index.ts +++ b/web/apps/photos/src/utils/image/index.ts @@ -1,8 +1,8 @@ // these utils only work in env where OffscreenCanvas is available import { Matrix, inverse } from "ml-matrix"; +import { FaceAlignment } from "services/ml/types"; import { BlobOptions, Dimensions } from "types/image"; -import { FaceAlignment } from "types/machineLearning"; import { enlargeBox } from "utils/machineLearning"; import { Box } from "../../../thirdparty/face-api/classes"; diff --git a/web/apps/photos/src/utils/machineLearning/config.ts b/web/apps/photos/src/utils/machineLearning/config.ts index 0c25356ab..f47a96b05 100644 --- a/web/apps/photos/src/utils/machineLearning/config.ts +++ b/web/apps/photos/src/utils/machineLearning/config.ts @@ -3,8 +3,8 @@ import { DEFAULT_ML_SYNC_CONFIG, DEFAULT_ML_SYNC_JOB_CONFIG, } from "constants/mlConfig"; +import { MLSearchConfig, MLSyncConfig } from "services/ml/types"; import { JobConfig } from "types/common/job"; -import { MLSearchConfig, MLSyncConfig } from "types/machineLearning"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, ML_SYNC_CONFIG_NAME, diff --git a/web/apps/photos/src/utils/machineLearning/faceAlign.ts b/web/apps/photos/src/utils/machineLearning/faceAlign.ts index beb98cea9..620d3fff3 100644 --- a/web/apps/photos/src/utils/machineLearning/faceAlign.ts +++ b/web/apps/photos/src/utils/machineLearning/faceAlign.ts @@ -1,6 +1,6 @@ import { Matrix } from "ml-matrix"; +import { FaceAlignment, FaceDetection } from "services/ml/types"; import { getSimilarityTransformation } from "similarity-transformation"; -import { FaceAlignment, FaceDetection } from "types/machineLearning"; import { Point } from "../../../thirdparty/face-api/classes"; const ARCFACE_LANDMARKS = [ diff --git a/web/apps/photos/src/utils/machineLearning/faceCrop.ts b/web/apps/photos/src/utils/machineLearning/faceCrop.ts index d437a942d..ef87f9f17 100644 --- a/web/apps/photos/src/utils/machineLearning/faceCrop.ts +++ b/web/apps/photos/src/utils/machineLearning/faceCrop.ts @@ -1,4 +1,4 @@ -import { FaceAlignment, FaceCrop, FaceCropConfig } from "types/machineLearning"; +import { FaceAlignment, FaceCrop, FaceCropConfig } from "services/ml/types"; import { cropWithRotation } from "utils/image"; import { enlargeBox } from "."; import { Box } from "../../../thirdparty/face-api/classes"; diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index bc9ae3974..f2186399b 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -4,8 +4,6 @@ import log from "@/next/log"; import PQueue from "p-queue"; import DownloadManager from "services/download"; import { getLocalFiles } from "services/fileService"; -import { EnteFile } from "types/file"; -import { Dimensions } from "types/image"; import { DetectedFace, Face, @@ -13,7 +11,9 @@ import { MlFileData, Person, Versioned, -} from "types/machineLearning"; +} from "services/ml/types"; +import { EnteFile } from "types/file"; +import { Dimensions } from "types/image"; import { getRenderableImage } from "utils/file"; import { clamp, warpAffineFloat32List } from "utils/image"; import mlIDbStorage from "utils/storage/mlIDbStorage"; diff --git a/web/apps/photos/src/utils/machineLearning/mldataMappers.ts b/web/apps/photos/src/utils/machineLearning/mldataMappers.ts index fb91420aa..111644821 100644 --- a/web/apps/photos/src/utils/machineLearning/mldataMappers.ts +++ b/web/apps/photos/src/utils/machineLearning/mldataMappers.ts @@ -1,10 +1,10 @@ import { + ClipEmbedding, Face, FaceDetection, Landmark, MlFileData, -} from "types/machineLearning"; -import { ClipEmbedding } from "types/machineLearning/data/clip"; +} from "services/ml/types"; export interface FileML extends ServerFileMl { updatedAt: number; diff --git a/web/apps/photos/src/utils/storage/mlIDbStorage.ts b/web/apps/photos/src/utils/storage/mlIDbStorage.ts index 766c3ac9a..c90832df3 100644 --- a/web/apps/photos/src/utils/storage/mlIDbStorage.ts +++ b/web/apps/photos/src/utils/storage/mlIDbStorage.ts @@ -15,8 +15,15 @@ import { openDB, } from "idb"; import isElectron from "is-electron"; -import { Face, MLLibraryData, MlFileData, Person } from "types/machineLearning"; -import { IndexStatus } from "types/machineLearning/ui"; +import { Face, MLLibraryData, MlFileData, Person } from "services/ml/types"; + +export interface IndexStatus { + outOfSyncFilesExists: boolean; + nSyncedFiles: number; + nTotalFiles: number; + localFilesSynced: boolean; + peopleIndexSynced: boolean; +} interface Config {} diff --git a/web/apps/photos/src/worker/ml.worker.ts b/web/apps/photos/src/worker/ml.worker.ts index ed46b7bd4..aa650d4af 100644 --- a/web/apps/photos/src/worker/ml.worker.ts +++ b/web/apps/photos/src/worker/ml.worker.ts @@ -1,8 +1,8 @@ import log from "@/next/log"; import { expose } from "comlink"; import mlService from "services/machineLearning/machineLearningService"; +import { MachineLearningWorker } from "services/ml/types"; import { EnteFile } from "types/file"; -import { MachineLearningWorker } from "types/machineLearning"; export class DedicatedMLWorker implements MachineLearningWorker { constructor() {