|
@@ -7,278 +7,12 @@ import "package:photos/models/file/file.dart";
|
|
|
import 'package:photos/models/ml/ml_typedefs.dart';
|
|
|
import "package:photos/models/ml/ml_versions.dart";
|
|
|
import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart';
|
|
|
-import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart';
|
|
|
import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';
|
|
|
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
|
|
|
import 'package:photos/services/machine_learning/face_ml/face_ml_methods.dart';
|
|
|
|
|
|
final _logger = Logger('ClusterResult_FaceMlResult');
|
|
|
|
|
|
-// TODO: should I add [faceMlVersion] and [clusterMlVersion] to the [ClusterResult] class?
|
|
|
-@Deprecated('We are now just storing the cluster results directly in DB')
|
|
|
-class ClusterResult {
|
|
|
- final int personId;
|
|
|
- String? userDefinedName;
|
|
|
- bool get hasUserDefinedName => userDefinedName != null;
|
|
|
-
|
|
|
- String _thumbnailFaceId;
|
|
|
- bool thumbnailFaceIdIsUserDefined;
|
|
|
-
|
|
|
- final List<int> _fileIds;
|
|
|
- final List<String> _faceIds;
|
|
|
-
|
|
|
- final Embedding medoid;
|
|
|
- double medoidDistanceThreshold;
|
|
|
-
|
|
|
- List<int> get uniqueFileIds => _fileIds.toSet().toList();
|
|
|
- List<int> get fileIDsIncludingPotentialDuplicates => _fileIds;
|
|
|
-
|
|
|
- List<String> get faceIDs => _faceIds;
|
|
|
-
|
|
|
- String get thumbnailFaceId => _thumbnailFaceId;
|
|
|
-
|
|
|
- int get thumbnailFileId => getFileIdFromFaceId(_thumbnailFaceId);
|
|
|
-
|
|
|
- /// Sets the thumbnail faceId to the given faceId.
|
|
|
- /// Throws an exception if the faceId is not in the list of faceIds.
|
|
|
- set setThumbnailFaceId(String faceId) {
|
|
|
- if (!_faceIds.contains(faceId)) {
|
|
|
- throw Exception(
|
|
|
- "The faceId $faceId is not in the list of faceIds: $faceId",
|
|
|
- );
|
|
|
- }
|
|
|
- _thumbnailFaceId = faceId;
|
|
|
- thumbnailFaceIdIsUserDefined = true;
|
|
|
- }
|
|
|
-
|
|
|
- /// Sets the [userDefinedName] to the given [customName]
|
|
|
- set setUserDefinedName(String customName) {
|
|
|
- userDefinedName = customName;
|
|
|
- }
|
|
|
-
|
|
|
- int get clusterSize => _fileIds.toSet().length;
|
|
|
-
|
|
|
- ClusterResult({
|
|
|
- required this.personId,
|
|
|
- required String thumbnailFaceId,
|
|
|
- required List<int> fileIds,
|
|
|
- required List<String> faceIds,
|
|
|
- required this.medoid,
|
|
|
- required this.medoidDistanceThreshold,
|
|
|
- this.userDefinedName,
|
|
|
- this.thumbnailFaceIdIsUserDefined = false,
|
|
|
- }) : _thumbnailFaceId = thumbnailFaceId,
|
|
|
- _faceIds = faceIds,
|
|
|
- _fileIds = fileIds;
|
|
|
-
|
|
|
- void addFileIDsAndFaceIDs(List<int> fileIDs, List<String> faceIDs) {
|
|
|
- assert(fileIDs.length == faceIDs.length);
|
|
|
- _fileIds.addAll(fileIDs);
|
|
|
- _faceIds.addAll(faceIDs);
|
|
|
- }
|
|
|
-
|
|
|
- // TODO: Consider if we should recalculated the medoid and threshold when deleting or adding a file from the cluster
|
|
|
- int removeFileId(int fileId) {
|
|
|
- assert(_fileIds.length == _faceIds.length);
|
|
|
- if (!_fileIds.contains(fileId)) {
|
|
|
- throw Exception(
|
|
|
- "The fileId $fileId is not in the list of fileIds: $fileId, so it's not in the cluster and cannot be removed.",
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- int removedCount = 0;
|
|
|
- for (var i = 0; i < _fileIds.length; i++) {
|
|
|
- if (_fileIds[i] == fileId) {
|
|
|
- assert(getFileIdFromFaceId(_faceIds[i]) == fileId);
|
|
|
- _fileIds.removeAt(i);
|
|
|
- _faceIds.removeAt(i);
|
|
|
- debugPrint(
|
|
|
- "Removed fileId $fileId from cluster $personId at index ${i + removedCount}}",
|
|
|
- );
|
|
|
- i--; // Adjust index due to removal
|
|
|
- removedCount++;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- _ensureClusterSizeIsAboveMinimum();
|
|
|
-
|
|
|
- return removedCount;
|
|
|
- }
|
|
|
-
|
|
|
- int addFileID(int fileID) {
|
|
|
- assert(_fileIds.length == _faceIds.length);
|
|
|
- if (_fileIds.contains(fileID)) {
|
|
|
- return 0;
|
|
|
- }
|
|
|
-
|
|
|
- _fileIds.add(fileID);
|
|
|
- _faceIds.add(FaceDetectionRelative.toFaceIDEmpty(fileID: fileID));
|
|
|
-
|
|
|
- return 1;
|
|
|
- }
|
|
|
-
|
|
|
- void ensureThumbnailFaceIdIsInCluster() {
|
|
|
- if (!_faceIds.contains(_thumbnailFaceId)) {
|
|
|
- _thumbnailFaceId = _faceIds[0];
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- void _ensureClusterSizeIsAboveMinimum() {
|
|
|
- if (clusterSize < minimumClusterSize) {
|
|
|
- throw Exception(
|
|
|
- "Cluster size is below minimum cluster size of $minimumClusterSize",
|
|
|
- );
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- Map<String, dynamic> _toJson() => {
|
|
|
- 'personId': personId,
|
|
|
- 'thumbnailFaceId': _thumbnailFaceId,
|
|
|
- 'fileIds': _fileIds,
|
|
|
- 'faceIds': _faceIds,
|
|
|
- 'medoid': medoid,
|
|
|
- 'medoidDistanceThreshold': medoidDistanceThreshold,
|
|
|
- if (userDefinedName != null) 'userDefinedName': userDefinedName,
|
|
|
- 'thumbnailFaceIdIsUserDefined': thumbnailFaceIdIsUserDefined,
|
|
|
- };
|
|
|
-
|
|
|
- String toJsonString() => jsonEncode(_toJson());
|
|
|
-
|
|
|
- static ClusterResult _fromJson(Map<String, dynamic> json) {
|
|
|
- return ClusterResult(
|
|
|
- personId: json['personId'] ?? -1,
|
|
|
- thumbnailFaceId: json['thumbnailFaceId'] ?? '',
|
|
|
- fileIds:
|
|
|
- (json['fileIds'] as List?)?.map((item) => item as int).toList() ?? [],
|
|
|
- faceIds:
|
|
|
- (json['faceIds'] as List?)?.map((item) => item as String).toList() ??
|
|
|
- [],
|
|
|
- medoid:
|
|
|
- (json['medoid'] as List?)?.map((item) => item as double).toList() ??
|
|
|
- [],
|
|
|
- medoidDistanceThreshold: json['medoidDistanceThreshold'] ?? 0,
|
|
|
- userDefinedName: json['userDefinedName'],
|
|
|
- thumbnailFaceIdIsUserDefined:
|
|
|
- json['thumbnailFaceIdIsUserDefined'] as bool,
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- static ClusterResult fromJsonString(String jsonString) {
|
|
|
- return _fromJson(jsonDecode(jsonString));
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-class ClusterResultBuilder {
|
|
|
- int personId = -1;
|
|
|
- String? userDefinedName;
|
|
|
- String thumbnailFaceId = '';
|
|
|
- bool thumbnailFaceIdIsUserDefined = false;
|
|
|
-
|
|
|
- List<int> fileIds = <int>[];
|
|
|
- List<String> faceIds = <String>[];
|
|
|
-
|
|
|
- List<Embedding> embeddings = <Embedding>[];
|
|
|
- Embedding medoid = <double>[];
|
|
|
- double medoidDistanceThreshold = 0;
|
|
|
- bool medoidAndThresholdCalculated = false;
|
|
|
- final int k = 5;
|
|
|
-
|
|
|
- ClusterResultBuilder.createFromIndices({
|
|
|
- required List<int> clusterIndices,
|
|
|
- required List<int> labels,
|
|
|
- required List<Embedding> allEmbeddings,
|
|
|
- required List<int> allFileIds,
|
|
|
- required List<String> allFaceIds,
|
|
|
- }) {
|
|
|
- final clusteredFileIds =
|
|
|
- clusterIndices.map((fileIndex) => allFileIds[fileIndex]).toList();
|
|
|
- final clusteredFaceIds =
|
|
|
- clusterIndices.map((fileIndex) => allFaceIds[fileIndex]).toList();
|
|
|
- final clusteredEmbeddings =
|
|
|
- clusterIndices.map((fileIndex) => allEmbeddings[fileIndex]).toList();
|
|
|
- personId = labels[clusterIndices[0]];
|
|
|
- fileIds = clusteredFileIds;
|
|
|
- faceIds = clusteredFaceIds;
|
|
|
- thumbnailFaceId = faceIds[0];
|
|
|
- embeddings = clusteredEmbeddings;
|
|
|
- }
|
|
|
-
|
|
|
- void calculateAndSetMedoidAndThreshold() {
|
|
|
- if (embeddings.isEmpty) {
|
|
|
- throw Exception("Cannot calculate medoid and threshold for empty list");
|
|
|
- }
|
|
|
-
|
|
|
- // Calculate the medoid and threshold
|
|
|
- final (tempMedoid, distanceThreshold) =
|
|
|
- _calculateMedoidAndDistanceTreshold(embeddings);
|
|
|
-
|
|
|
- // Update the medoid
|
|
|
- medoid = List.from(tempMedoid);
|
|
|
-
|
|
|
- // Update the medoidDistanceThreshold as the distance of the medoid to its k-th nearest neighbor
|
|
|
- medoidDistanceThreshold = distanceThreshold;
|
|
|
-
|
|
|
- medoidAndThresholdCalculated = true;
|
|
|
- }
|
|
|
-
|
|
|
- (List<double>, double) _calculateMedoidAndDistanceTreshold(
|
|
|
- List<List<double>> embeddings,
|
|
|
- ) {
|
|
|
- double minDistance = double.infinity;
|
|
|
- List<double>? medoid;
|
|
|
-
|
|
|
- // Calculate the distance between all pairs
|
|
|
- for (int i = 0; i < embeddings.length; ++i) {
|
|
|
- double totalDistance = 0;
|
|
|
- for (int j = 0; j < embeddings.length; ++j) {
|
|
|
- if (i != j) {
|
|
|
- totalDistance += cosineDistance(embeddings[i], embeddings[j]);
|
|
|
-
|
|
|
- // Break early if we already exceed minDistance
|
|
|
- if (totalDistance > minDistance) {
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // Find the minimum total distance
|
|
|
- if (totalDistance < minDistance) {
|
|
|
- minDistance = totalDistance;
|
|
|
- medoid = embeddings[i];
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // Now, calculate k-th nearest neighbor for the medoid
|
|
|
- final List<double> distancesToMedoid = [];
|
|
|
- for (List<double> embedding in embeddings) {
|
|
|
- if (embedding != medoid) {
|
|
|
- distancesToMedoid.add(cosineDistance(medoid!, embedding));
|
|
|
- }
|
|
|
- }
|
|
|
- distancesToMedoid.sort();
|
|
|
- // TODO: empirically find the best k. Probably it should be dynamic in some way, so for instance larger for larger clusters and smaller for smaller clusters, especially since there are a lot of really small clusters and a few really large ones.
|
|
|
- final double kthDistance = distancesToMedoid[
|
|
|
- distancesToMedoid.length >= k ? k - 1 : distancesToMedoid.length - 1];
|
|
|
-
|
|
|
- return (medoid!, kthDistance);
|
|
|
- }
|
|
|
-
|
|
|
- void changeThumbnailFaceId(String faceId) {
|
|
|
- if (!faceIds.contains(faceId)) {
|
|
|
- throw Exception(
|
|
|
- "The faceId $faceId is not in the list of faceIds: $faceIds",
|
|
|
- );
|
|
|
- }
|
|
|
- thumbnailFaceId = faceId;
|
|
|
- }
|
|
|
-
|
|
|
- void addFileIDsAndFaceIDs(List<int> addedFileIDs, List<String> addedFaceIDs) {
|
|
|
- assert(addedFileIDs.length == addedFaceIDs.length);
|
|
|
- fileIds.addAll(addedFileIDs);
|
|
|
- faceIds.addAll(addedFaceIDs);
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
@immutable
|
|
|
class FaceMlResult {
|
|
|
final int fileId;
|