diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart index bb03de519..6530cde13 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_result.dart @@ -7,278 +7,12 @@ import "package:photos/models/file/file.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; import "package:photos/models/ml/ml_versions.dart"; import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; -import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:photos/services/machine_learning/face_ml/face_ml_methods.dart'; final _logger = Logger('ClusterResult_FaceMlResult'); -// TODO: should I add [faceMlVersion] and [clusterMlVersion] to the [ClusterResult] class? -@Deprecated('We are now just storing the cluster results directly in DB') -class ClusterResult { - final int personId; - String? userDefinedName; - bool get hasUserDefinedName => userDefinedName != null; - - String _thumbnailFaceId; - bool thumbnailFaceIdIsUserDefined; - - final List _fileIds; - final List _faceIds; - - final Embedding medoid; - double medoidDistanceThreshold; - - List get uniqueFileIds => _fileIds.toSet().toList(); - List get fileIDsIncludingPotentialDuplicates => _fileIds; - - List get faceIDs => _faceIds; - - String get thumbnailFaceId => _thumbnailFaceId; - - int get thumbnailFileId => getFileIdFromFaceId(_thumbnailFaceId); - - /// Sets the thumbnail faceId to the given faceId. - /// Throws an exception if the faceId is not in the list of faceIds. - set setThumbnailFaceId(String faceId) { - if (!_faceIds.contains(faceId)) { - throw Exception( - "The faceId $faceId is not in the list of faceIds: $faceId", - ); - } - _thumbnailFaceId = faceId; - thumbnailFaceIdIsUserDefined = true; - } - - /// Sets the [userDefinedName] to the given [customName] - set setUserDefinedName(String customName) { - userDefinedName = customName; - } - - int get clusterSize => _fileIds.toSet().length; - - ClusterResult({ - required this.personId, - required String thumbnailFaceId, - required List fileIds, - required List faceIds, - required this.medoid, - required this.medoidDistanceThreshold, - this.userDefinedName, - this.thumbnailFaceIdIsUserDefined = false, - }) : _thumbnailFaceId = thumbnailFaceId, - _faceIds = faceIds, - _fileIds = fileIds; - - void addFileIDsAndFaceIDs(List fileIDs, List faceIDs) { - assert(fileIDs.length == faceIDs.length); - _fileIds.addAll(fileIDs); - _faceIds.addAll(faceIDs); - } - - // TODO: Consider if we should recalculated the medoid and threshold when deleting or adding a file from the cluster - int removeFileId(int fileId) { - assert(_fileIds.length == _faceIds.length); - if (!_fileIds.contains(fileId)) { - throw Exception( - "The fileId $fileId is not in the list of fileIds: $fileId, so it's not in the cluster and cannot be removed.", - ); - } - - int removedCount = 0; - for (var i = 0; i < _fileIds.length; i++) { - if (_fileIds[i] == fileId) { - assert(getFileIdFromFaceId(_faceIds[i]) == fileId); - _fileIds.removeAt(i); - _faceIds.removeAt(i); - debugPrint( - "Removed fileId $fileId from cluster $personId at index ${i + removedCount}}", - ); - i--; // Adjust index due to removal - removedCount++; - } - } - - _ensureClusterSizeIsAboveMinimum(); - - return removedCount; - } - - int addFileID(int fileID) { - assert(_fileIds.length == _faceIds.length); - if (_fileIds.contains(fileID)) { - return 0; - } - - _fileIds.add(fileID); - _faceIds.add(FaceDetectionRelative.toFaceIDEmpty(fileID: fileID)); - - return 1; - } - - void ensureThumbnailFaceIdIsInCluster() { - if (!_faceIds.contains(_thumbnailFaceId)) { - _thumbnailFaceId = _faceIds[0]; - } - } - - void _ensureClusterSizeIsAboveMinimum() { - if (clusterSize < minimumClusterSize) { - throw Exception( - "Cluster size is below minimum cluster size of $minimumClusterSize", - ); - } - } - - Map _toJson() => { - 'personId': personId, - 'thumbnailFaceId': _thumbnailFaceId, - 'fileIds': _fileIds, - 'faceIds': _faceIds, - 'medoid': medoid, - 'medoidDistanceThreshold': medoidDistanceThreshold, - if (userDefinedName != null) 'userDefinedName': userDefinedName, - 'thumbnailFaceIdIsUserDefined': thumbnailFaceIdIsUserDefined, - }; - - String toJsonString() => jsonEncode(_toJson()); - - static ClusterResult _fromJson(Map json) { - return ClusterResult( - personId: json['personId'] ?? -1, - thumbnailFaceId: json['thumbnailFaceId'] ?? '', - fileIds: - (json['fileIds'] as List?)?.map((item) => item as int).toList() ?? [], - faceIds: - (json['faceIds'] as List?)?.map((item) => item as String).toList() ?? - [], - medoid: - (json['medoid'] as List?)?.map((item) => item as double).toList() ?? - [], - medoidDistanceThreshold: json['medoidDistanceThreshold'] ?? 0, - userDefinedName: json['userDefinedName'], - thumbnailFaceIdIsUserDefined: - json['thumbnailFaceIdIsUserDefined'] as bool, - ); - } - - static ClusterResult fromJsonString(String jsonString) { - return _fromJson(jsonDecode(jsonString)); - } -} - -class ClusterResultBuilder { - int personId = -1; - String? userDefinedName; - String thumbnailFaceId = ''; - bool thumbnailFaceIdIsUserDefined = false; - - List fileIds = []; - List faceIds = []; - - List embeddings = []; - Embedding medoid = []; - double medoidDistanceThreshold = 0; - bool medoidAndThresholdCalculated = false; - final int k = 5; - - ClusterResultBuilder.createFromIndices({ - required List clusterIndices, - required List labels, - required List allEmbeddings, - required List allFileIds, - required List allFaceIds, - }) { - final clusteredFileIds = - clusterIndices.map((fileIndex) => allFileIds[fileIndex]).toList(); - final clusteredFaceIds = - clusterIndices.map((fileIndex) => allFaceIds[fileIndex]).toList(); - final clusteredEmbeddings = - clusterIndices.map((fileIndex) => allEmbeddings[fileIndex]).toList(); - personId = labels[clusterIndices[0]]; - fileIds = clusteredFileIds; - faceIds = clusteredFaceIds; - thumbnailFaceId = faceIds[0]; - embeddings = clusteredEmbeddings; - } - - void calculateAndSetMedoidAndThreshold() { - if (embeddings.isEmpty) { - throw Exception("Cannot calculate medoid and threshold for empty list"); - } - - // Calculate the medoid and threshold - final (tempMedoid, distanceThreshold) = - _calculateMedoidAndDistanceTreshold(embeddings); - - // Update the medoid - medoid = List.from(tempMedoid); - - // Update the medoidDistanceThreshold as the distance of the medoid to its k-th nearest neighbor - medoidDistanceThreshold = distanceThreshold; - - medoidAndThresholdCalculated = true; - } - - (List, double) _calculateMedoidAndDistanceTreshold( - List> embeddings, - ) { - double minDistance = double.infinity; - List? medoid; - - // Calculate the distance between all pairs - for (int i = 0; i < embeddings.length; ++i) { - double totalDistance = 0; - for (int j = 0; j < embeddings.length; ++j) { - if (i != j) { - totalDistance += cosineDistance(embeddings[i], embeddings[j]); - - // Break early if we already exceed minDistance - if (totalDistance > minDistance) { - break; - } - } - } - - // Find the minimum total distance - if (totalDistance < minDistance) { - minDistance = totalDistance; - medoid = embeddings[i]; - } - } - - // Now, calculate k-th nearest neighbor for the medoid - final List distancesToMedoid = []; - for (List embedding in embeddings) { - if (embedding != medoid) { - distancesToMedoid.add(cosineDistance(medoid!, embedding)); - } - } - distancesToMedoid.sort(); - // TODO: empirically find the best k. Probably it should be dynamic in some way, so for instance larger for larger clusters and smaller for smaller clusters, especially since there are a lot of really small clusters and a few really large ones. - final double kthDistance = distancesToMedoid[ - distancesToMedoid.length >= k ? k - 1 : distancesToMedoid.length - 1]; - - return (medoid!, kthDistance); - } - - void changeThumbnailFaceId(String faceId) { - if (!faceIds.contains(faceId)) { - throw Exception( - "The faceId $faceId is not in the list of faceIds: $faceIds", - ); - } - thumbnailFaceId = faceId; - } - - void addFileIDsAndFaceIDs(List addedFileIDs, List addedFaceIDs) { - assert(addedFileIDs.length == addedFaceIDs.length); - fileIds.addAll(addedFileIDs); - faceIds.addAll(addedFaceIDs); - } -} - @immutable class FaceMlResult { final int fileId;