import React, { useState, useEffect, useContext, ChangeEvent } from 'react'; import { getData, LS_KEYS } from 'utils/storage/localStorage'; import { useRouter } from 'next/router'; import { ComlinkWorker } from 'utils/comlink'; import { AppContext } from 'pages/_app'; import { PAGES } from 'constants/pages'; import * as Comlink from 'comlink'; import { runningInBrowser } from 'utils/common'; import TFJSImage from './TFJSImage'; import { Face, FACE_CROPS_CACHE_NAME, MLDebugResult, MLSyncConfig, Person, } from 'types/machineLearning'; import Tree from 'react-d3-tree'; import MLFileDebugView from './MLFileDebugView'; import mlWorkManager from 'services/machineLearning/mlWorkManager'; // import { getAllFacesMap, mlLibraryStore } from 'utils/storage/mlStorage'; import { getAllFacesFromMap, getAllPeople } from 'utils/machineLearning'; import { FaceImagesRow, ImageBlobView, ImageCacheView } from './ImageViews'; import mlIDbStorage from 'utils/storage/mlIDbStorage'; import { getFaceCropBlobFromStorage } from 'utils/machineLearning/faceCrop'; import { PeopleList } from './PeopleList'; import styled from 'styled-components'; import { RawNodeDatum } from 'react-d3-tree/lib/types/common'; import { DebugInfo, mstToBinaryTree } from 'hdbscan'; import { toD3Tree } from 'utils/machineLearning/clustering'; import { getMLSyncConfig, getMLSyncJobConfig, updateMLSyncConfig, updateMLSyncJobConfig, } from 'utils/machineLearning/config'; import { Button, Col, Container, Form, Row } from 'react-bootstrap'; import { JobConfig } from 'types/common/job'; import { ConfigEditor } from './ConfigEditor'; import { DEFAULT_ML_SYNC_CONFIG, DEFAULT_ML_SYNC_JOB_CONFIG, } from 'constants/machineLearning/config'; import { exportMlData, importMlData } from 'utils/machineLearning/mldataExport'; interface TSNEProps { mlResult: MLDebugResult; } function TSNEPlot(props: TSNEProps) { return ( {props.mlResult.tsne.dataset.map((data, i) => ( ))} ); } const D3ImageContainer = styled.div` & > img { width: 100%; height: 100%; } `; const renderForeignObjectNode = ({ nodeDatum, foreignObjectProps }) => ( {/* `foreignObject` requires width & height to be explicitly set. */}

{nodeDatum.name}

{!nodeDatum.children && nodeDatum.name && ( )}
); const getFaceCrops = async (faces: Face[]) => { const faceCropPromises = faces .filter((f) => f?.crop) .map((f) => getFaceCropBlobFromStorage(f.crop)); return Promise.all(faceCropPromises); }; const ClusterFacesRow = styled(FaceImagesRow)` display: flex; max-width: 100%; overflow: auto; `; const RowWithGap = styled(Row)` justify-content: center; & > * { margin: 10px; } `; export default function MLDebug() { const [token, setToken] = useState(); const [clusterFaceDistance] = useState(0.4); // const [minClusterSize, setMinClusterSize] = useState(5); // const [minFaceSize, setMinFaceSize] = useState(32); // const [batchSize, setBatchSize] = useState(200); const [maxFaceDistance] = useState(0.5); const [mlResult, setMlResult] = useState({ allFaces: [], clustersWithNoise: { clusters: [], noise: [], }, tree: null, tsne: null, }); const [allPeople, setAllPeople] = useState>([]); const [clusters, setClusters] = useState>>([]); const [noiseFaces, setNoiseFaces] = useState>([]); const [minProbability, setMinProbability] = useState(0); const [maxProbability, setMaxProbability] = useState(1); const [filteredFaces, setFilteredFaces] = useState>([]); const [mstD3Tree, setMstD3Tree] = useState(null); const [debugFile, setDebugFile] = useState(); const router = useRouter(); const appContext = useContext(AppContext); const getDedicatedMLWorker = (): ComlinkWorker => { if (token) { console.log('Toen present'); } if (runningInBrowser()) { console.log('initiating worker'); const worker = new Worker( new URL('worker/machineLearning.worker', import.meta.url), { name: 'ml-worker' } ); console.log('initiated worker'); const comlink = Comlink.wrap(worker); return { comlink, worker }; } }; let MLWorker: ComlinkWorker; useEffect(() => { const user = getData(LS_KEYS.USER); if (!user?.token) { router.push(PAGES.ROOT); } else { setToken(user.token); } appContext.showNavBar(true); }, []); const onSync = async () => { try { if (!MLWorker) { MLWorker = getDedicatedMLWorker(); console.log('initiated MLWorker'); } const mlWorker = await new MLWorker.comlink(); const result = await mlWorker.sync( token, clusterFaceDistance, // minClusterSize, // minFaceSize, // batchSize, maxFaceDistance ); setMlResult(result); } catch (e) { console.error(e); throw e; } finally { // setTimeout(()=>{ // console.log('terminating ml-worker'); MLWorker.worker.terminate(); // }, 30000); } }; const onStartMLSync = async () => { mlWorkManager.startSyncJob(); }; const onStopMLSync = async () => { mlWorkManager.stopSyncJob(); }; // for debug purpose, not a memory efficient implementation const onExportMLData = async () => { let mlDataZipHandle: FileSystemFileHandle; try { mlDataZipHandle = await showSaveFilePicker({ suggestedName: `ente-mldata-${Date.now()}`, types: [ { accept: { 'application/zip': ['.zip'], }, }, ], }); } catch (e) { console.error(e); return; } try { const mlDataZipWritable = await mlDataZipHandle.createWritable(); await exportMlData(mlDataZipWritable); } catch (e) { console.error('Error while exporting: ', e); } }; const onImportMLData = async () => { let mlDataZipHandle: FileSystemFileHandle; try { [mlDataZipHandle] = await showOpenFilePicker({ types: [ { accept: { 'application/zip': ['.zip'], }, }, ], }); } catch (e) { console.error(e); return; } try { const mlDataZipFile = await mlDataZipHandle.getFile(); await importMlData(mlDataZipFile); } catch (e) { console.error('Error while importing: ', e); } }; const onClearPeopleIndex = async () => { mlIDbStorage.setIndexVersion('people', 0); }; const onDebugFile = async (event: ChangeEvent) => { setDebugFile(event.target.files[0]); }; const onLoadAllPeople = async () => { const allPeople = await getAllPeople(100); setAllPeople(allPeople); }; const onLoadClusteringResults = async () => { const mlLibraryData = await mlIDbStorage.getLibraryData(); const allFacesMap = await mlIDbStorage.getAllFacesMap(); const allFaces = getAllFacesFromMap(allFacesMap); const clusterPromises = mlLibraryData?.faceClusteringResults?.clusters .map((cluster) => cluster?.slice(0, 200).map((f) => allFaces[f])) .map((faces) => getFaceCrops(faces)); setClusters(await Promise.all(clusterPromises)); const noiseFaces = mlLibraryData?.faceClusteringResults?.noise ?.slice(0, 200) .map((n) => allFaces[n]); setNoiseFaces(await getFaceCrops(noiseFaces)); // TODO: disabling mst binary tree display for faces > 1000 // can enable once toD3Tree is non recursive // and only important part of tree is retrieved const clusteringDebugInfo: DebugInfo = mlLibraryData?.faceClusteringResults['debugInfo']; if (allFaces.length <= 1000 && clusteringDebugInfo) { const mstBinaryTree = mstToBinaryTree(clusteringDebugInfo.mst); const d3Tree = toD3Tree(mstBinaryTree, allFaces); setMstD3Tree(d3Tree); } }; const showFilteredFaces = async () => { console.log('Filtering with: ', minProbability, maxProbability); const allFacesMap = await mlIDbStorage.getAllFacesMap(); const allFaces = getAllFacesFromMap(allFacesMap); const filteredFaces = allFaces .filter( (f) => f.detection.probability >= minProbability && f.detection.probability <= maxProbability ) .slice(0, 200); setFilteredFaces(await getFaceCrops(filteredFaces)); }; const nodeSize = { x: 180, y: 180 }; const foreignObjectProps = { width: 112, height: 150, x: -56 }; // TODO: Remove debug page or config editor from prod return ( {/*
ClusterFaceDistance: {clusterFaceDistance}

*/}
getMLSyncConfig()} defaultConfig={() => Promise.resolve(DEFAULT_ML_SYNC_CONFIG) } setConfig={(mlSyncConfig) => updateMLSyncConfig(mlSyncConfig as MLSyncConfig) }> getMLSyncJobConfig()} defaultConfig={() => Promise.resolve(DEFAULT_ML_SYNC_JOB_CONFIG) } setConfig={(mlSyncJobConfig) => updateMLSyncJobConfig(mlSyncJobConfig as JobConfig) }> {/*
MinFaceSize: {minFaceSize}

MinClusterSize: {minClusterSize}

Number of Images in Batch: {batchSize}
*/} {/*

MaxFaceDistance: {maxFaceDistance}
*/}


All identified people:
Clusters: {clusters.map((cluster, index) => ( {cluster?.map((face, i) => ( ))} ))}

Noise: {noiseFaces?.map((face, i) => ( ))}
Show Faces based on detection probability: Min: setMinProbability( (parseFloat(e.target.value) || 0) / 100 ) } /> Max: setMaxProbability( (parseFloat(e.target.value) || 100) / 100 ) } />

{filteredFaces?.map((face, i) => ( ))}
Debug File:
Hdbscan MST:
{mstD3Tree && ( renderForeignObjectNode({ ...rd3tProps, foreignObjectProps, }) } /> )}

TSNE of embeddings:
{mlResult.tsne && }
); }