diff --git a/web/apps/photos/src/services/face/cluster.ts b/web/apps/photos/src/services/face/cluster.ts new file mode 100644 index 000000000..9ddf156cc --- /dev/null +++ b/web/apps/photos/src/services/face/cluster.ts @@ -0,0 +1,34 @@ +import { Hdbscan, type DebugInfo } from "hdbscan"; +import { type Cluster } from "services/face/types"; + +export interface ClusterFacesResult { + clusters: Array; + noise: Cluster; + debugInfo?: DebugInfo; +} + +/** + * Cluster the given {@link faceEmbeddings}. + * + * @param faceEmbeddings An array of embeddings produced by our face indexing + * pipeline. Each embedding is for a face detected in an image (a single image + * may have multiple faces detected within it). + */ +export const clusterFaces = async ( + faceEmbeddings: Array>, +): Promise => { + const hdbscan = new Hdbscan({ + input: faceEmbeddings, + minClusterSize: 3, + minSamples: 5, + clusterSelectionEpsilon: 0.6, + clusterSelectionMethod: "leaf", + debug: true, + }); + + return { + clusters: hdbscan.getClusters(), + noise: hdbscan.getNoise(), + debugInfo: hdbscan.getDebugInfo(), + }; +}; diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index b652657dd..f7a88d9c6 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -1,5 +1,5 @@ -import { DebugInfo } from "hdbscan"; import PQueue from "p-queue"; +import type { ClusterFacesResult } from "services/face/cluster"; import { Dimensions } from "services/face/geom"; import { EnteFile } from "types/file"; import { Box, Point } from "./geom"; @@ -17,15 +17,6 @@ export declare type FaceDescriptor = Float32Array; export declare type Cluster = Array; -export interface ClusteringResults { - clusters: Array; - noise: Cluster; -} - -export interface HdbscanResults extends ClusteringResults { - debugInfo?: DebugInfo; -} - export interface FacesCluster { faces: Cluster; summary?: FaceDescriptor; @@ -212,7 +203,6 @@ export interface MLSyncContext { faceCropService: FaceCropService; faceEmbeddingService: FaceEmbeddingService; blurDetectionService: BlurDetectionService; - faceClusteringService: ClusteringService; localFilesMap: Map; outOfSyncFiles: EnteFile[]; @@ -246,7 +236,7 @@ export interface MLSyncFileContext { export interface MLLibraryData { faceClusteringMethod?: Versioned; - faceClusteringResults?: ClusteringResults; + faceClusteringResults?: ClusterFacesResult; faceClustersWithNoise?: FacesClustersWithNoise; } @@ -283,14 +273,6 @@ export interface BlurDetectionService { detectBlur(alignedFaces: Float32Array, faces: Face[]): number[]; } -export interface ClusteringService { - method: Versioned; - - cluster(input: ClusteringInput): Promise; -} - -export declare type ClusteringInput = Array>; - export interface MachineLearningWorker { closeLocalSyncContext(): Promise; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 7183db1f1..8b521add9 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -11,6 +11,7 @@ import { type Versioned, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; +import { clusterFaces } from "../face/cluster"; import { fetchImageBitmap, fetchImageBitmapForContext, @@ -257,12 +258,13 @@ class FaceService { } log.info("Running clustering allFaces: ", allFaces.length); - syncContext.mlLibraryData.faceClusteringResults = - await syncContext.faceClusteringService.cluster( - allFaces.map((f) => Array.from(f.embedding)), - ); - syncContext.mlLibraryData.faceClusteringMethod = - syncContext.faceClusteringService.method; + syncContext.mlLibraryData.faceClusteringResults = await clusterFaces( + allFaces.map((f) => Array.from(f.embedding)), + ); + syncContext.mlLibraryData.faceClusteringMethod = { + value: "Hdbscan", + version: 1, + }; log.info( "[MLService] Got face clustering results: ", JSON.stringify(syncContext.mlLibraryData.faceClusteringResults), diff --git a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts deleted file mode 100644 index 1f508d545..000000000 --- a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Hdbscan } from "hdbscan"; -import { - ClusteringInput, - ClusteringMethod, - ClusteringService, - HdbscanResults, - Versioned, -} from "services/face/types"; - -class HdbscanClusteringService implements ClusteringService { - public method: Versioned; - - constructor() { - this.method = { - value: "Hdbscan", - version: 1, - }; - } - - public async cluster(input: ClusteringInput): Promise { - // log.info('Clustering input: ', input); - const hdbscan = new Hdbscan({ - input, - - minClusterSize: 3, - minSamples: 5, - clusterSelectionEpsilon: 0.6, - clusterSelectionMethod: "leaf", - debug: true, - }); - - return { - clusters: hdbscan.getClusters(), - noise: hdbscan.getNoise(), - debugInfo: hdbscan.getDebugInfo(), - }; - } -} - -export default new HdbscanClusteringService(); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index eb3d50558..a944e4cf2 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -14,8 +14,6 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db"; import { BlurDetectionMethod, BlurDetectionService, - ClusteringMethod, - ClusteringService, Face, FaceCropMethod, FaceCropService, @@ -38,7 +36,6 @@ import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import arcfaceCropService from "./arcfaceCropService"; import FaceService from "./faceService"; -import hdbscanClusteringService from "./hdbscanClusteringService"; import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; import PeopleService from "./peopleService"; @@ -155,16 +152,6 @@ export class MLFactory { throw Error("Unknon face embedding method: " + method); } - - public static getClusteringService( - method: ClusteringMethod, - ): ClusteringService { - if (method === "Hdbscan") { - return hdbscanClusteringService; - } - - throw Error("Unknon clustering method: " + method); - } } export class LocalMLSyncContext implements MLSyncContext { @@ -176,7 +163,6 @@ export class LocalMLSyncContext implements MLSyncContext { public faceCropService: FaceCropService; public blurDetectionService: BlurDetectionService; public faceEmbeddingService: FaceEmbeddingService; - public faceClusteringService: ClusteringService; public localFilesMap: Map; public outOfSyncFiles: EnteFile[]; @@ -215,7 +201,6 @@ export class LocalMLSyncContext implements MLSyncContext { MLFactory.getBlurDetectionService("Laplacian"); this.faceEmbeddingService = MLFactory.getFaceEmbeddingService("MobileFaceNet"); - this.faceClusteringService = MLFactory.getClusteringService("Hdbscan"); this.outOfSyncFiles = []; this.nSyncedFiles = 0; diff --git a/web/apps/photos/src/services/machineLearning/peopleService.ts b/web/apps/photos/src/services/machineLearning/peopleService.ts index c485a62be..2224f4200 100644 --- a/web/apps/photos/src/services/machineLearning/peopleService.ts +++ b/web/apps/photos/src/services/machineLearning/peopleService.ts @@ -9,10 +9,10 @@ class PeopleService { const filesVersion = await mlIDbStorage.getIndexVersion("files"); if ( filesVersion <= (await mlIDbStorage.getIndexVersion("people")) && - !isDifferentOrOld( - syncContext.mlLibraryData?.faceClusteringMethod, - syncContext.faceClusteringService.method, - ) + !isDifferentOrOld(syncContext.mlLibraryData?.faceClusteringMethod, { + value: "Hdbscan", + version: 1, + }) ) { log.info( "[MLService] Skipping people index as already synced to latest version",