diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index 8732296ab..0be17f3e2 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -16,7 +16,6 @@ "@tensorflow/tfjs-converter": "^4.10.0", "@tensorflow/tfjs-core": "^4.10.0", "@tensorflow/tfjs-tflite": "0.0.1-alpha.7", - "@zip.js/zip.js": "2.4.2", "bip39": "^3.0.4", "blazeface-back": "^0.0.9", "bootstrap": "^4.5.2", diff --git a/web/apps/photos/src/components/ml-debug/index.tsx b/web/apps/photos/src/components/ml-debug/index.tsx deleted file mode 100644 index a5b53a684..000000000 --- a/web/apps/photos/src/components/ml-debug/index.tsx +++ /dev/null @@ -1,12 +0,0 @@ -// import dynamic from 'next/dynamic'; - -// const MLDebugWithNoSSR = dynamic( -// () => import('components/MachineLearning/MlDebug-disabled'), -// { -// ssr: false, -// } -// ); - -export default function MLDebug() { - return
{/* */}
; -} diff --git a/web/apps/photos/src/utils/machineLearning/clustering.ts b/web/apps/photos/src/utils/machineLearning/clustering.ts deleted file mode 100644 index c630715b1..000000000 --- a/web/apps/photos/src/utils/machineLearning/clustering.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { euclidean } from "hdbscan"; -// import { RawNodeDatum } from 'react-d3-tree/lib/types/common'; -// import { f32Average, getAllFacesFromMap } from '.'; -import { addLogLine } from "@ente/shared/logging"; -import { - FacesCluster, - FaceWithEmbedding, - MLSyncContext, - NearestCluster, -} from "types/machineLearning"; - -export function updateClusterSummaries(syncContext: MLSyncContext) { - if ( - !syncContext.mlLibraryData?.faceClusteringResults?.clusters || - syncContext.mlLibraryData?.faceClusteringResults?.clusters.length < 1 - ) { - return; - } - - const resultClusters = - syncContext.mlLibraryData.faceClusteringResults.clusters; - - resultClusters.forEach((resultCluster) => { - syncContext.mlLibraryData.faceClustersWithNoise.clusters.push({ - faces: resultCluster, - // summary: this.getClusterSummary(resultCluster), - }); - }); -} - -export function getNearestCluster( - syncContext: MLSyncContext, - noise: FaceWithEmbedding, -): NearestCluster { - let nearest: FacesCluster = null; - let nearestDist = 100000; - syncContext.mlLibraryData.faceClustersWithNoise.clusters.forEach((c) => { - const dist = euclidean( - Array.from(noise.embedding), - Array.from(c.summary), - ); - if (dist < nearestDist) { - nearestDist = dist; - nearest = c; - } - }); - - addLogLine("nearestDist: ", nearestDist); - return { cluster: nearest, distance: nearestDist }; -} - -// export async function assignNoiseWithinLimit(syncContext: MLSyncContext) { -// if ( -// !syncContext.mlLibraryData?.faceClusteringResults?.noise || -// syncContext.mlLibraryData?.faceClusteringResults.noise.length < 1 -// ) { -// return; -// } - -// const noise = syncContext.mlLibraryData.faceClusteringResults.noise; -// const allFacesMap = await getAllFacesMap(); -// const allFaces = getAllFacesFromMap(allFacesMap); - -// noise.forEach((n) => { -// const noiseFace = allFaces[n]; -// const nearest = this.getNearestCluster(syncContext, noiseFace); - -// if (nearest.cluster && nearest.distance < this.maxFaceDistance) { -// addLogLine('Adding noise to cluser: ', n, nearest.distance); -// nearest.cluster.faces.push(n); -// } else { -// addLogLine( -// 'No cluster for noise: ', -// n, -// 'within distance: ', -// this.maxFaceDistance -// ); -// this.clustersWithNoise.noise.push(n); -// } -// }); -// } - -// TODO: remove recursion to avoid stack size limits -// export function toD3Tree( -// treeNode: TreeNode, -// allObjects: Array -// ): RawNodeDatum { -// if (!treeNode.left && !treeNode.right) { -// return { -// name: treeNode.data.toString(), -// attributes: { -// face: allObjects[treeNode.data], -// }, -// }; -// } -// const children = []; -// treeNode.left && children.push(toD3Tree(treeNode.left, allObjects)); -// treeNode.right && children.push(toD3Tree(treeNode.right, allObjects)); - -// return { -// name: treeNode.data.toString(), -// children: children, -// }; -// } diff --git a/web/apps/photos/src/utils/machineLearning/compatibility.ts b/web/apps/photos/src/utils/machineLearning/compatibility.ts deleted file mode 100644 index 47916707d..000000000 --- a/web/apps/photos/src/utils/machineLearning/compatibility.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { - offscreenCanvasSupported, - runningInChrome, - webglSupported, -} from "utils/common"; - -import { addLogLine } from "@ente/shared/logging"; -import isElectron from "is-electron"; - -export function canEnableMlSearch(): boolean { - // check if is chrome or ente desktop - if (!runningInChrome(false) && !isElectron()) { - addLogLine("Not running in Chrome Desktop or Ente Desktop App"); - return false; - } - - if (!offscreenCanvasSupported()) { - addLogLine("OffscreenCanvas is NOT supported"); - return false; - } - - if (!webglSupported()) { - addLogLine("webgl is NOT supported"); - return false; - } - - return true; -} diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index d6be9e63f..57845937b 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -1,10 +1,15 @@ +import { addLogLine } from "@ente/shared/logging"; +import { CACHES } from "@ente/shared/storage/cacheStorage/constants"; +import { cached } from "@ente/shared/storage/cacheStorage/helpers"; import * as tf from "@tensorflow/tfjs-core"; import { NormalizedFace } from "blazeface-back"; +import { FILE_TYPE } from "constants/file"; import { BLAZEFACE_FACE_SIZE } from "constants/mlConfig"; import { euclidean } from "hdbscan"; import PQueue from "p-queue"; import DownloadManager from "services/download"; import { getLocalFiles } from "services/fileService"; +import { decodeLivePhoto } from "services/livePhotoService"; import { EnteFile } from "types/file"; import { Dimensions } from "types/image"; import { @@ -18,12 +23,6 @@ import { RealWorldObject, Versioned, } from "types/machineLearning"; -// import { mlFilesStore, mlPeopleStore } from 'utils/storage/mlStorage'; -import { addLogLine } from "@ente/shared/logging"; -import { CACHES } from "@ente/shared/storage/cacheStorage/constants"; -import { cached } from "@ente/shared/storage/cacheStorage/helpers"; -import { FILE_TYPE } from "constants/file"; -import { decodeLivePhoto } from "services/livePhotoService"; import { getRenderableImage } from "utils/file"; import { imageBitmapToBlob } from "utils/image"; import mlIDbStorage from "utils/storage/mlIDbStorage"; diff --git a/web/apps/photos/src/utils/machineLearning/mldataExport.ts b/web/apps/photos/src/utils/machineLearning/mldataExport.ts deleted file mode 100644 index 4cd115e12..000000000 --- a/web/apps/photos/src/utils/machineLearning/mldataExport.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { addLogLine } from "@ente/shared/logging"; -import { CacheStorageService } from "@ente/shared/storage/cacheStorage"; -import { CACHES } from "@ente/shared/storage/cacheStorage/constants"; -import * as zip from "@zip.js/zip.js"; -import { MlFileData } from "types/machineLearning"; -import mlIDbStorage from "utils/storage/mlIDbStorage"; - -class FileSystemWriter extends zip.Writer { - writableStream: FileSystemWritableFileStream; - - constructor(writableStream: FileSystemWritableFileStream) { - super(); - this.writableStream = writableStream; - } - - async writeUint8Array(array: Uint8Array) { - // addLogLine('zipWriter needs to write data: ', array.byteLength); - return this.writableStream.write(array); - } - - async getData() { - return undefined; - } -} - -class FileReader extends zip.Reader { - file: File; - - constructor(file: File) { - super(); - this.file = file; - } - - public async init() { - this.size = this.file.size; - // addLogLine('zipReader init, size: ', this.size); - } - - public async readUint8Array( - index: number, - length: number, - ): Promise { - // addLogLine('zipReader needs data: ', index, length); - const slicedFile = this.file.slice(index, index + length); - const arrayBuffer = await slicedFile.arrayBuffer(); - - return new Uint8Array(arrayBuffer); - } -} - -export async function exportMlData( - mlDataZipWritable: FileSystemWritableFileStream, -) { - const zipWriter = new zip.ZipWriter( - new FileSystemWriter(mlDataZipWritable), - ); - - try { - try { - await exportMlDataToZipWriter(zipWriter); - } finally { - await zipWriter.close(); - } - } catch (e) { - await mlDataZipWritable.abort(); - throw e; - } - - await mlDataZipWritable.close(); - addLogLine("Ml Data Exported"); -} - -async function exportMlDataToZipWriter(zipWriter: zip.ZipWriter) { - const mlDbData = await mlIDbStorage.getAllMLData(); - const faceClusteringResults = - mlDbData?.library?.data?.faceClusteringResults; - faceClusteringResults && (faceClusteringResults.debugInfo = undefined); - addLogLine( - "Exporting ML DB data: ", - JSON.stringify(Object.keys(mlDbData)), - JSON.stringify( - Object.keys(mlDbData)?.map((k) => Object.keys(mlDbData[k])?.length), - ), - ); - await zipWriter.add( - "indexeddb/mldata.json", - new zip.TextReader(JSON.stringify(mlDbData)), - ); - - const faceCropCache = await CacheStorageService.open(CACHES.FACE_CROPS); - const files = - mlDbData["files"] && (Object.values(mlDbData["files"]) as MlFileData[]); - for (const fileData of files || []) { - for (const face of fileData.faces || []) { - const faceCropUrl = face.crop?.imageUrl; - if (!faceCropUrl) { - console.error("face crop not found for faceId: ", face.id); - continue; - } - const response = await faceCropCache.match(faceCropUrl); - if (response && response.ok) { - const blob = await response.blob(); - await zipWriter.add( - `caches/${CACHES.FACE_CROPS}${faceCropUrl}`, - new zip.BlobReader(blob), - { level: 0 }, - ); - } else { - console.error( - "face crop cache entry not found for faceCropUrl: ", - faceCropUrl, - ); - } - } - } -} -export async function importMlData(mlDataZipFile: File) { - const zipReader = new zip.ZipReader(new FileReader(mlDataZipFile)); - - try { - await importMlDataFromZipReader(zipReader); - } finally { - await zipReader.close(); - } - - addLogLine("ML Data Imported"); -} - -async function importMlDataFromZipReader(zipReader: zip.ZipReader) { - const zipEntries = await zipReader.getEntries(); - // addLogLine(zipEntries); - - const faceCropPath = `caches/${CACHES.FACE_CROPS}`; - const faceCropCache = await CacheStorageService.open(CACHES.FACE_CROPS); - let mldataEntry; - for (const entry of zipEntries) { - if (entry.filename === "indexeddb/mldata.json") { - mldataEntry = entry; - } else if (entry.filename.startsWith(faceCropPath)) { - const faceCropUrl = entry.filename.substring(faceCropPath.length); - // addLogLine('importing faceCropUrl: ', faceCropUrl); - const faceCropCacheBlob: Blob = await entry.getData( - new zip.BlobWriter("image/jpeg"), - ); - faceCropCache.put(faceCropUrl, new Response(faceCropCacheBlob)); - } - } - - const mlDataJsonStr: string = await mldataEntry.getData( - new zip.TextWriter(), - ); - const mlDbData = JSON.parse(mlDataJsonStr); - addLogLine( - "importing ML DB data: ", - JSON.stringify(Object.keys(mlDbData)), - JSON.stringify( - Object.keys(mlDbData)?.map((k) => Object.keys(mlDbData[k])?.length), - ), - ); - await mlIDbStorage.putAllMLData(mlDbData); -} diff --git a/web/apps/photos/src/worker/ml.worker.ts b/web/apps/photos/src/worker/ml.worker.ts index a66188d43..33c2e5583 100644 --- a/web/apps/photos/src/worker/ml.worker.ts +++ b/web/apps/photos/src/worker/ml.worker.ts @@ -3,23 +3,12 @@ import { expose } from "comlink"; import mlService from "services/machineLearning/machineLearningService"; import { EnteFile } from "types/file"; import { MachineLearningWorker } from "types/machineLearning"; -// import ReverseProxiedElectronCacheStorageProxy from './electronCacheStorageProxy.proxy'; -// import { setupResponseComlinkTransferHandler } from 'utils/comlink'; export class DedicatedMLWorker implements MachineLearningWorker { constructor() { addLogLine("DedicatedMLWorker constructor called"); - // this.init(); } - // public async init() { - // const recp = new ReverseProxiedElectronCacheStorageProxy(); - // const cacheProxy = await recp.open('thumbs'); - - // const thumb = await cacheProxy.match('13578875'); - // addLogLine('worker init cache.match', thumb); - // } - public async closeLocalSyncContext() { return mlService.closeLocalSyncContext(); } @@ -51,5 +40,3 @@ export class DedicatedMLWorker implements MachineLearningWorker { } expose(DedicatedMLWorker, self); - -// setupResponseComlinkTransferHandler(); diff --git a/web/yarn.lock b/web/yarn.lock index 0be0fe1c0..af80a9a98 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -947,11 +947,6 @@ resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.38.tgz#6fda4b410edc753d3213c648320ebcf319669020" integrity sha512-7LrhVKz2PRh+DD7+S+PVaFd5HxaWQvoMqBbsV9fNJO1pjUs1P8bM2vQVNfk+3URTqbuTI7gkXi0rfsN0IadoBA== -"@zip.js/zip.js@2.4.2": - version "2.4.2" - resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.4.2.tgz#7c2d4b381c0f73dffa1d02ef630c4168ebd5cc4a" - integrity sha512-D4xr9g7U625Q01lQASr9g1sQWEGhndyd+G3v3OvY/qH3pwaJDpbVDy+TO6yEq7c8teQdjjTiiBvlcQcVtT+itg== - acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" @@ -2828,7 +2823,7 @@ libsodium-wrappers@0.7.9: dependencies: libsodium "^0.7.0" -libsodium@^0.7.0: +libsodium@0.7.9, libsodium@^0.7.0: version "0.7.9" resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.9.tgz#4bb7bcbf662ddd920d8795c227ae25bbbfa3821b" integrity sha512-gfeADtR4D/CM0oRUviKBViMGXZDgnFdMKMzHsvBdqLBHd9ySi6EtYnmuhHVDDYgYpAO8eU8hEY+F8vIUAPh08A==