Merge branch 'mobile_face' of https://github.com/ente-io/auth into mobile_face

This commit is contained in:
Neeraj Gupta 2024-04-20 16:01:08 +05:30
commit cc682a0a09
21 changed files with 1259 additions and 547 deletions

View file

@ -1316,8 +1316,8 @@ class FilesDB {
}
Future<Map<int, int>> getFileIDToCreationTime() async {
final db = await instance.database;
final rows = await db.rawQuery(
final db = await instance.sqliteAsyncDB;
final rows = await db.getAll(
'''
SELECT $columnUploadedFileID, $columnCreationTime
FROM $filesTable

View file

@ -27,4 +27,5 @@ enum EventType {
unhide,
coverChanged,
peopleChanged,
peopleClusterChanged,
}

View file

@ -1,3 +1,22 @@
import "package:photos/events/event.dart";
import "package:photos/models/file/file.dart";
class PeopleChangedEvent extends Event {}
class PeopleChangedEvent extends Event {
final List<EnteFile>? relevantFiles;
final PeopleEventType type;
final String source;
PeopleChangedEvent({
this.relevantFiles,
this.type = PeopleEventType.defaultType,
this.source = "",
});
@override
String get reason => '$runtimeType{type: ${type.name}, "via": $source}';
}
enum PeopleEventType {
defaultType,
removedFilesFromCluster,
}

View file

@ -12,6 +12,7 @@ import 'package:photos/face/db_fields.dart';
import "package:photos/face/db_model_mappers.dart";
import "package:photos/face/model/face.dart";
import "package:photos/models/file/file.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
import 'package:sqflite/sqflite.dart';
import 'package:sqlite_async/sqlite_async.dart' as sqlite_async;
@ -160,27 +161,27 @@ class FaceMLDataDB {
final db = await instance.database;
// find out clusterIds that are assigned to other persons using the clusters table
final List<Map<String, dynamic>> maps = await db.rawQuery(
'SELECT $cluserIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL',
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL',
[personID],
);
final Set<int> ignoredClusterIDs =
maps.map((e) => e[cluserIDColumn] as int).toSet();
maps.map((e) => e[clusterIDColumn] as int).toSet();
final List<Map<String, dynamic>> rejectMaps = await db.rawQuery(
'SELECT $cluserIDColumn FROM $notPersonFeedback WHERE $personIdColumn = ?',
'SELECT $clusterIDColumn FROM $notPersonFeedback WHERE $personIdColumn = ?',
[personID],
);
final Set<int> rejectClusterIDs =
rejectMaps.map((e) => e[cluserIDColumn] as int).toSet();
rejectMaps.map((e) => e[clusterIDColumn] as int).toSet();
return ignoredClusterIDs.union(rejectClusterIDs);
}
Future<Set<int>> getPersonClusterIDs(String personID) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
'SELECT $cluserIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?',
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?',
[personID],
);
return maps.map((e) => e[cluserIDColumn] as int).toSet();
return maps.map((e) => e[clusterIDColumn] as int).toSet();
}
Future<void> clearTable() async {
@ -249,16 +250,16 @@ class FaceMLDataDB {
}
final cluterRows = await db.query(
clusterPersonTable,
columns: [cluserIDColumn],
columns: [clusterIDColumn],
where: '$personIdColumn = ?',
whereArgs: [personID],
);
final clusterIDs =
cluterRows.map((e) => e[cluserIDColumn] as int).toList();
cluterRows.map((e) => e[clusterIDColumn] as int).toList();
final List<Map<String, dynamic>> faceMaps = await db.rawQuery(
'SELECT * FROM $facesTable where '
'$faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID IN (${clusterIDs.join(",")}))'
'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinHighQualityFaceScore ORDER BY $faceScore DESC',
'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinimumQualityFaceScore ORDER BY $faceScore DESC',
);
if (faceMaps.isNotEmpty) {
if (avatarFileId != null) {
@ -308,8 +309,6 @@ class FaceMLDataDB {
faceBlur,
imageHeight,
imageWidth,
faceArea,
faceVisibilityScore,
mlVersionColumn,
],
where: '$fileIDColumn = ?',
@ -334,16 +333,60 @@ class FaceMLDataDB {
}
Future<Iterable<String>> getFaceIDsForCluster(int clusterID) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.query(
faceClustersTable,
columns: [fcFaceId],
where: '$fcClusterID = ?',
whereArgs: [clusterID],
final db = await instance.sqliteAsyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable '
'WHERE $faceClustersTable.$fcClusterID = ?',
[clusterID],
);
return maps.map((e) => e[fcFaceId] as String).toSet();
}
Future<Iterable<String>> getFaceIDsForPerson(String personID) async {
final db = await instance.sqliteAsyncDB;
final faceIdsResult = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
'WHERE $clusterPersonTable.$personIdColumn = ?',
[personID],
);
return faceIdsResult.map((e) => e[fcFaceId] as String).toSet();
}
Future<Iterable<double>> getBlurValuesForCluster(int clusterID) async {
final db = await instance.sqliteAsyncDB;
const String query = '''
SELECT $facesTable.$faceBlur
FROM $facesTable
JOIN $faceClustersTable ON $facesTable.$faceIDColumn = $faceClustersTable.$fcFaceId
WHERE $faceClustersTable.$fcClusterID = ?
''';
// const String query2 = '''
// SELECT $faceBlur
// FROM $facesTable
// WHERE $faceIDColumn IN (SELECT $fcFaceId FROM $faceClustersTable WHERE $fcClusterID = ?)
// ''';
final List<Map<String, dynamic>> maps = await db.getAll(
query,
[clusterID],
);
return maps.map((e) => e[faceBlur] as double).toSet();
}
Future<Map<String, double>> getFaceIDsToBlurValues(
int maxBlurValue,
) async {
final db = await instance.sqliteAsyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceIDColumn, $faceBlur FROM $facesTable WHERE $faceBlur < $maxBlurValue AND $faceBlur > 1 ORDER BY $faceBlur ASC',
);
final Map<String, double> result = {};
for (final map in maps) {
result[map[faceIDColumn] as String] = map[faceBlur] as double;
}
return result;
}
Future<Map<String, int?>> getFaceIdsToClusterIds(
Iterable<String> faceIds,
) async {
@ -376,14 +419,14 @@ class FaceMLDataDB {
}
Future<void> forceUpdateClusterIds(
Map<String, int> faceIDToPersonID,
Map<String, int> faceIDToClusterID,
) async {
final db = await instance.database;
// Start a batch
final batch = db.batch();
for (final map in faceIDToPersonID.entries) {
for (final map in faceIDToClusterID.entries) {
final faceID = map.key;
final clusterID = map.value;
batch.insert(
@ -410,12 +453,64 @@ class FaceMLDataDB {
);
}
Future<Set<FaceInfoForClustering>> getFaceInfoForClustering({
double minScore = kMinimumQualityFaceScore,
int minClarity = kLaplacianHardThreshold,
int maxFaces = 20000,
int offset = 0,
int batchSize = 10000,
}) async {
final EnteWatch w = EnteWatch("getFaceEmbeddingMap")..start();
w.logAndReset(
'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
);
final db = await instance.sqliteAsyncDB;
final Set<FaceInfoForClustering> result = {};
while (true) {
// Query a batch of rows
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceIDColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways FROM $facesTable'
' WHERE $faceScore > $minScore AND $faceBlur > $minClarity'
' ORDER BY $faceIDColumn'
' DESC LIMIT $batchSize OFFSET $offset',
);
// Break the loop if no more rows
if (maps.isEmpty) {
break;
}
final List<String> faceIds = [];
for (final map in maps) {
faceIds.add(map[faceIDColumn] as String);
}
final faceIdToClusterId = await getFaceIdsToClusterIds(faceIds);
for (final map in maps) {
final faceID = map[faceIDColumn] as String;
final faceInfo = FaceInfoForClustering(
faceID: faceID,
clusterId: faceIdToClusterId[faceID],
embeddingBytes: map[faceEmbeddingBlob] as Uint8List,
faceScore: map[faceScore] as double,
blurValue: map[faceBlur] as double,
isSideways: (map[isSideways] as int) == 1,
);
result.add(faceInfo);
}
if (result.length >= maxFaces) {
break;
}
offset += batchSize;
}
w.stopWithLog('done reading face embeddings ${result.length}');
return result;
}
/// Returns a map of faceID to record of clusterId and faceEmbeddingBlob
///
/// Only selects faces with score greater than [minScore] and blur score greater than [minClarity]
Future<Map<String, (int?, Uint8List)>> getFaceEmbeddingMap({
double minScore = kMinHighQualityFaceScore,
int minClarity = kLaplacianThreshold,
double minScore = kMinimumQualityFaceScore,
int minClarity = kLaplacianHardThreshold,
int maxFaces = 20000,
int offset = 0,
int batchSize = 10000,
@ -481,7 +576,7 @@ class FaceMLDataDB {
facesTable,
columns: [faceIDColumn, faceEmbeddingBlob],
where:
'$faceScore > $kMinHighQualityFaceScore AND $faceBlur > $kLaplacianThreshold AND $fileIDColumn IN (${fileIDs.join(",")})',
'$faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})',
limit: batchSize,
offset: offset,
orderBy: '$faceIDColumn DESC',
@ -503,12 +598,50 @@ class FaceMLDataDB {
return result;
}
Future<Map<String, Uint8List>> getFaceEmbeddingMapForFaces(
Iterable<String> faceIDs,
) async {
_logger.info('reading face embeddings for ${faceIDs.length} faces');
final db = await instance.sqliteAsyncDB;
// Define the batch size
const batchSize = 10000;
int offset = 0;
final Map<String, Uint8List> result = {};
while (true) {
// Query a batch of rows
final String query = '''
SELECT $faceIDColumn, $faceEmbeddingBlob
FROM $facesTable
WHERE $faceIDColumn IN (${faceIDs.map((id) => "'$id'").join(",")})
ORDER BY $faceIDColumn DESC
LIMIT $batchSize OFFSET $offset
''';
final List<Map<String, dynamic>> maps = await db.getAll(query);
// Break the loop if no more rows
if (maps.isEmpty) {
break;
}
for (final map in maps) {
final faceID = map[faceIDColumn] as String;
result[faceID] = map[faceEmbeddingBlob] as Uint8List;
}
if (result.length > 10000) {
break;
}
offset += batchSize;
}
_logger.info('done reading face embeddings for ${faceIDs.length} faces');
return result;
}
Future<int> getTotalFaceCount({
double minFaceScore = kMinHighQualityFaceScore,
double minFaceScore = kMinimumQualityFaceScore,
}) async {
final db = await instance.sqliteAsyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianThreshold',
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianHardThreshold',
);
return maps.first['count'] as int;
}
@ -517,7 +650,7 @@ class FaceMLDataDB {
final db = await instance.sqliteAsyncDB;
final List<Map<String, dynamic>> totalFacesMaps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinHighQualityFaceScore AND $faceBlur > $kLaplacianThreshold',
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold',
);
final int totalFaces = totalFacesMaps.first['count'] as int;
@ -530,11 +663,11 @@ class FaceMLDataDB {
}
Future<int> getBlurryFaceCount([
int blurThreshold = kLaplacianThreshold,
int blurThreshold = kLaplacianHardThreshold,
]) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceBlur <= $blurThreshold AND $faceScore > $kMinHighQualityFaceScore',
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceBlur <= $blurThreshold AND $faceScore > $kMinimumQualityFaceScore',
);
return maps.first['count'] as int;
}
@ -555,7 +688,7 @@ class FaceMLDataDB {
clusterPersonTable,
{
personIdColumn: personID,
cluserIDColumn: clusterID,
clusterIDColumn: clusterID,
},
);
}
@ -572,7 +705,7 @@ class FaceMLDataDB {
clusterPersonTable,
{
personIdColumn: personID,
cluserIDColumn: clusterID,
clusterIDColumn: clusterID,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
@ -589,11 +722,31 @@ class FaceMLDataDB {
notPersonFeedback,
{
personIdColumn: personID,
cluserIDColumn: clusterID,
clusterIDColumn: clusterID,
},
);
}
Future<void> bulkCaptureNotPersonFeedback(
Map<int, String> clusterToPersonID,
) async {
final db = await instance.database;
final batch = db.batch();
for (final entry in clusterToPersonID.entries) {
final clusterID = entry.key;
final personID = entry.value;
batch.insert(
notPersonFeedback,
{
personIdColumn: personID,
clusterIDColumn: clusterID,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
await batch.commit(noResult: true);
}
Future<int> removeClusterToPerson({
required String personID,
required int clusterID,
@ -601,7 +754,7 @@ class FaceMLDataDB {
final db = await instance.database;
return db.delete(
clusterPersonTable,
where: '$personIdColumn = ? AND $cluserIDColumn = ?',
where: '$personIdColumn = ? AND $clusterIDColumn = ?',
whereArgs: [personID, clusterID],
);
}
@ -613,13 +766,13 @@ class FaceMLDataDB {
final List<Map<String, dynamic>> maps = await db.rawQuery(
'SELECT $faceClustersTable.$fcClusterID, $fcFaceId FROM $faceClustersTable '
'INNER JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$cluserIDColumn '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
'WHERE $clusterPersonTable.$personIdColumn = ?',
[personID],
);
final Map<int, Set<int>> result = {};
for (final map in maps) {
final clusterID = map[cluserIDColumn] as int;
final clusterID = map[clusterIDColumn] as int;
final String faceID = map[fcFaceId] as String;
final fileID = int.parse(faceID.split('_').first);
result[fileID] = (result[fileID] ?? {})..add(clusterID);
@ -664,7 +817,7 @@ class FaceMLDataDB {
batch.insert(
clusterSummaryTable,
{
cluserIDColumn: cluserID,
clusterIDColumn: cluserID,
avgColumn: avg,
countColumn: count,
},
@ -676,12 +829,16 @@ class FaceMLDataDB {
}
/// Returns a map of clusterID to (avg embedding, count)
Future<Map<int, (Uint8List, int)>> clusterSummaryAll() async {
final db = await instance.database;
Future<Map<int, (Uint8List, int)>> getAllClusterSummary([
int? minClusterSize,
]) async {
final db = await instance.sqliteAsyncDB;
final Map<int, (Uint8List, int)> result = {};
final rows = await db.rawQuery('SELECT * from $clusterSummaryTable');
final rows = await db.getAll(
'SELECT * FROM $clusterSummaryTable${minClusterSize != null ? ' WHERE $countColumn >= $minClusterSize' : ''}',
);
for (final r in rows) {
final id = r[cluserIDColumn] as int;
final id = r[clusterIDColumn] as int;
final avg = r[avgColumn] as Uint8List;
final count = r[countColumn] as int;
result[id] = (avg, count);
@ -692,11 +849,11 @@ class FaceMLDataDB {
Future<Map<int, String>> getClusterIDToPersonID() async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
'SELECT $personIdColumn, $cluserIDColumn FROM $clusterPersonTable',
'SELECT $personIdColumn, $clusterIDColumn FROM $clusterPersonTable',
);
final Map<int, String> result = {};
for (final map in maps) {
result[map[cluserIDColumn] as int] = map[personIdColumn] as String;
result[map[clusterIDColumn] as int] = map[personIdColumn] as String;
}
return result;
}
@ -741,7 +898,7 @@ class FaceMLDataDB {
final db = await instance.database;
final faceIdsResult = await db.rawQuery(
'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$cluserIDColumn '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
'WHERE $clusterPersonTable.$personIdColumn = ?',
[personID],
);

View file

@ -8,8 +8,7 @@ const faceDetectionColumn = 'detection';
const faceEmbeddingBlob = 'eBlob';
const faceScore = 'score';
const faceBlur = 'blur';
const faceArea = 'area';
const faceVisibilityScore = 'visibility';
const isSideways = 'is_sideways';
const imageWidth = 'width';
const imageHeight = 'height';
const faceClusterId = 'cluster_id';
@ -22,10 +21,9 @@ const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable (
$faceEmbeddingBlob BLOB NOT NULL,
$faceScore REAL NOT NULL,
$faceBlur REAL NOT NULL DEFAULT $kLapacianDefault,
$isSideways INTEGER NOT NULL DEFAULT 0,
$imageHeight INTEGER NOT NULL DEFAULT 0,
$imageWidth INTEGER NOT NULL DEFAULT 0,
$faceArea INTEGER NOT NULL DEFAULT 0,
$faceVisibilityScore INTEGER NOT NULL DEFAULT -1,
$mlVersionColumn INTEGER NOT NULL DEFAULT -1,
PRIMARY KEY($fileIDColumn, $faceIDColumn)
);
@ -62,13 +60,13 @@ const deletePersonTable = 'DROP TABLE IF EXISTS $personTable';
// Clusters Table Fields & Schema Queries
const clusterPersonTable = 'cluster_person';
const personIdColumn = 'person_id';
const cluserIDColumn = 'cluster_id';
const clusterIDColumn = 'cluster_id';
const createClusterPersonTable = '''
CREATE TABLE IF NOT EXISTS $clusterPersonTable (
$personIdColumn TEXT NOT NULL,
$cluserIDColumn INTEGER NOT NULL,
PRIMARY KEY($personIdColumn, $cluserIDColumn)
$clusterIDColumn INTEGER NOT NULL,
PRIMARY KEY($personIdColumn, $clusterIDColumn)
);
''';
const dropClusterPersonTable = 'DROP TABLE IF EXISTS $clusterPersonTable';
@ -80,10 +78,10 @@ const avgColumn = 'avg';
const countColumn = 'count';
const createClusterSummaryTable = '''
CREATE TABLE IF NOT EXISTS $clusterSummaryTable (
$cluserIDColumn INTEGER NOT NULL,
$clusterIDColumn INTEGER NOT NULL,
$avgColumn BLOB NOT NULL,
$countColumn INTEGER NOT NULL,
PRIMARY KEY($cluserIDColumn)
PRIMARY KEY($clusterIDColumn)
);
''';
@ -97,7 +95,7 @@ const notPersonFeedback = 'not_person_feedback';
const createNotPersonFeedbackTable = '''
CREATE TABLE IF NOT EXISTS $notPersonFeedback (
$personIdColumn TEXT NOT NULL,
$cluserIDColumn INTEGER NOT NULL
$clusterIDColumn INTEGER NOT NULL
);
''';
const dropNotPersonFeedbackTable = 'DROP TABLE IF EXISTS $notPersonFeedback';

View file

@ -34,9 +34,8 @@ Map<String, dynamic> mapRemoteToFaceDB(Face face) {
).writeToBuffer(),
faceScore: face.score,
faceBlur: face.blur,
isSideways: face.detection.faceIsSideways() ? 1 : 0,
mlVersionColumn: faceMlVersion,
faceArea: face.area(),
faceVisibilityScore: face.visibility,
imageWidth: face.fileInfo?.imageWidth ?? 0,
imageHeight: face.fileInfo?.imageHeight ?? 0,
};

View file

@ -1,6 +1,9 @@
import "dart:math" show min, max;
import "package:logging/logging.dart";
import "package:photos/face/model/box.dart";
import "package:photos/face/model/landmark.dart";
import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart";
/// Stores the face detection data, notably the bounding box and landmarks.
///
@ -19,7 +22,7 @@ class Detection {
bool get isEmpty => box.width == 0 && box.height == 0 && landmarks.isEmpty;
// emoty box
// empty box
Detection.empty()
: box = FaceBox(
xMin: 0,
@ -89,4 +92,72 @@ class Detection {
return -1;
}
}
FaceDirection getFaceDirection() {
if (isEmpty) {
return FaceDirection.straight;
}
final leftEye = [landmarks[0].x, landmarks[0].y];
final rightEye = [landmarks[1].x, landmarks[1].y];
final nose = [landmarks[2].x, landmarks[2].y];
final leftMouth = [landmarks[3].x, landmarks[3].y];
final rightMouth = [landmarks[4].x, landmarks[4].y];
final double eyeDistanceX = (rightEye[0] - leftEye[0]).abs();
final double eyeDistanceY = (rightEye[1] - leftEye[1]).abs();
final double mouthDistanceY = (rightMouth[1] - leftMouth[1]).abs();
final bool faceIsUpright =
(max(leftEye[1], rightEye[1]) + 0.5 * eyeDistanceY < nose[1]) &&
(nose[1] + 0.5 * mouthDistanceY < min(leftMouth[1], rightMouth[1]));
final bool noseStickingOutLeft = (nose[0] < min(leftEye[0], rightEye[0])) &&
(nose[0] < min(leftMouth[0], rightMouth[0]));
final bool noseStickingOutRight =
(nose[0] > max(leftEye[0], rightEye[0])) &&
(nose[0] > max(leftMouth[0], rightMouth[0]));
final bool noseCloseToLeftEye =
(nose[0] - leftEye[0]).abs() < 0.2 * eyeDistanceX;
final bool noseCloseToRightEye =
(nose[0] - rightEye[0]).abs() < 0.2 * eyeDistanceX;
// if (faceIsUpright && (noseStickingOutLeft || noseCloseToLeftEye)) {
if (noseStickingOutLeft || (faceIsUpright && noseCloseToLeftEye)) {
return FaceDirection.left;
// } else if (faceIsUpright && (noseStickingOutRight || noseCloseToRightEye)) {
} else if (noseStickingOutRight || (faceIsUpright && noseCloseToRightEye)) {
return FaceDirection.right;
}
return FaceDirection.straight;
}
bool faceIsSideways() {
if (isEmpty) {
return false;
}
final leftEye = [landmarks[0].x, landmarks[0].y];
final rightEye = [landmarks[1].x, landmarks[1].y];
final nose = [landmarks[2].x, landmarks[2].y];
final leftMouth = [landmarks[3].x, landmarks[3].y];
final rightMouth = [landmarks[4].x, landmarks[4].y];
final double eyeDistanceX = (rightEye[0] - leftEye[0]).abs();
final double eyeDistanceY = (rightEye[1] - leftEye[1]).abs();
final double mouthDistanceY = (rightMouth[1] - leftMouth[1]).abs();
final bool faceIsUpright =
(max(leftEye[1], rightEye[1]) + 0.5 * eyeDistanceY < nose[1]) &&
(nose[1] + 0.5 * mouthDistanceY < min(leftMouth[1], rightMouth[1]));
final bool noseStickingOutLeft =
(nose[0] < min(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) &&
(nose[0] < min(leftMouth[0], rightMouth[0]));
final bool noseStickingOutRight =
(nose[0] > max(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) &&
(nose[0] > max(leftMouth[0], rightMouth[0]));
return faceIsUpright && (noseStickingOutLeft || noseStickingOutRight);
}
}

View file

@ -20,9 +20,9 @@ class Face {
final double blur;
FileInfo? fileInfo;
bool get isBlurry => blur < kLaplacianThreshold;
bool get isBlurry => blur < kLaplacianHardThreshold;
bool get hasHighScore => score > kMinHighQualityFaceScore;
bool get hasHighScore => score > kMinimumQualityFaceScore;
bool get isHighQuality => (!isBlurry) && hasHighScore;

View file

@ -2,19 +2,26 @@ import "dart:async";
import "dart:developer";
import "dart:isolate";
import "dart:math" show max;
import "dart:typed_data";
import "dart:typed_data" show Uint8List;
import "package:computer/computer.dart";
import "package:flutter/foundation.dart" show kDebugMode;
import "package:logging/logging.dart";
import "package:ml_linalg/dtype.dart";
import "package:ml_linalg/vector.dart";
import "package:photos/generated/protos/ente/common/vector.pb.dart";
import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart';
import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/face_ml_result.dart";
import "package:simple_cluster/simple_cluster.dart";
import "package:synchronized/synchronized.dart";
class FaceInfo {
final String faceID;
final double? faceScore;
final double? blurValue;
final bool? badFace;
final List<double>? embedding;
final Vector? vEmbedding;
int? clusterId;
@ -23,6 +30,9 @@ class FaceInfo {
int? fileCreationTime;
FaceInfo({
required this.faceID,
this.faceScore,
this.blurValue,
this.badFace,
this.embedding,
this.vEmbedding,
this.clusterId,
@ -32,8 +42,18 @@ class FaceInfo {
enum ClusterOperation { linearIncrementalClustering, dbscanClustering }
class ClusteringResult {
final Map<String, int> newFaceIdToCluster;
final Map<int, (Uint8List, int)>? newClusterSummaries;
ClusteringResult({
required this.newFaceIdToCluster,
required this.newClusterSummaries,
});
}
class FaceClusteringService {
final _logger = Logger("FaceLinearClustering");
final _computer = Computer.shared();
Timer? _inactivityTimer;
final Duration _inactivityDuration = const Duration(minutes: 3);
@ -49,6 +69,7 @@ class FaceClusteringService {
bool isRunning = false;
static const kRecommendedDistanceThreshold = 0.24;
static const kConservativeDistanceThreshold = 0.06;
// singleton pattern
FaceClusteringService._privateConstructor();
@ -100,31 +121,11 @@ class FaceClusteringService {
try {
switch (function) {
case ClusterOperation.linearIncrementalClustering:
final input = args['input'] as Map<String, (int?, Uint8List)>;
final fileIDToCreationTime =
args['fileIDToCreationTime'] as Map<int, int>?;
final distanceThreshold = args['distanceThreshold'] as double;
final offset = args['offset'] as int?;
final result = FaceClusteringService._runLinearClustering(
input,
fileIDToCreationTime: fileIDToCreationTime,
distanceThreshold: distanceThreshold,
offset: offset,
);
final result = FaceClusteringService.runLinearClustering(args);
sendPort.send(result);
break;
case ClusterOperation.dbscanClustering:
final input = args['input'] as Map<String, Uint8List>;
final fileIDToCreationTime =
args['fileIDToCreationTime'] as Map<int, int>?;
final eps = args['eps'] as double;
final minPts = args['minPts'] as int;
final result = FaceClusteringService._runDbscanClustering(
input,
fileIDToCreationTime: fileIDToCreationTime,
eps: eps,
minPts: minPts,
);
final result = FaceClusteringService._runDbscanClustering(args);
sendPort.send(result);
break;
}
@ -194,16 +195,19 @@ class FaceClusteringService {
_inactivityTimer?.cancel();
}
/// Runs the clustering algorithm on the given [input], in an isolate.
/// Runs the clustering algorithm [runLinearClustering] on the given [input], in an isolate.
///
/// Returns the clustering result, which is a list of clusters, where each cluster is a list of indices of the dataset.
///
/// WARNING: Make sure to always input data in the same ordering, otherwise the clustering can less less deterministic.
Future<Map<String, int>?> predictLinear(
Map<String, (int?, Uint8List)> input, {
Future<ClusteringResult?> predictLinear(
Set<FaceInfoForClustering> input, {
Map<int, int>? fileIDToCreationTime,
double distanceThreshold = kRecommendedDistanceThreshold,
double conservativeDistanceThreshold = kConservativeDistanceThreshold,
bool useDynamicThreshold = true,
int? offset,
required Map<int, (Uint8List, int)> oldClusterSummaries,
}) async {
if (input.isEmpty) {
_logger.warning(
@ -225,20 +229,23 @@ class FaceClusteringService {
final stopwatchClustering = Stopwatch()..start();
// final Map<String, int> faceIdToCluster =
// await _runLinearClusteringInComputer(input);
final Map<String, int> faceIdToCluster = await _runInIsolate(
final ClusteringResult? faceIdToCluster = await _runInIsolate(
(
ClusterOperation.linearIncrementalClustering,
{
'input': input,
'fileIDToCreationTime': fileIDToCreationTime,
'distanceThreshold': distanceThreshold,
'conservativeDistanceThreshold': conservativeDistanceThreshold,
'useDynamicThreshold': useDynamicThreshold,
'offset': offset,
'oldClusterSummaries': oldClusterSummaries,
}
),
);
// return _runLinearClusteringInComputer(input);
_logger.info(
'Clustering executed in ${stopwatchClustering.elapsed.inSeconds} seconds',
'predictLinear Clustering executed in ${stopwatchClustering.elapsed.inSeconds} seconds',
);
isRunning = false;
@ -250,6 +257,142 @@ class FaceClusteringService {
}
}
/// Runs the clustering algorithm [runLinearClustering] on the given [input], in computer, without any dynamic thresholding
Future<ClusteringResult?> predictLinearComputer(
Map<String, Uint8List> input, {
Map<int, int>? fileIDToCreationTime,
double distanceThreshold = kRecommendedDistanceThreshold,
}) async {
if (input.isEmpty) {
_logger.warning(
"Linear Clustering dataset of embeddings is empty, returning empty list.",
);
return null;
}
// Clustering inside the isolate
_logger.info(
"Start Linear clustering on ${input.length} embeddings inside computer isolate",
);
try {
final clusteringInput = input
.map((key, value) {
return MapEntry(
key,
FaceInfoForClustering(
faceID: key,
embeddingBytes: value,
faceScore: kMinimumQualityFaceScore + 0.01,
blurValue: kLapacianDefault,
),
);
})
.values
.toSet();
final startTime = DateTime.now();
final faceIdToCluster = await _computer.compute(
runLinearClustering,
param: {
"input": clusteringInput,
"fileIDToCreationTime": fileIDToCreationTime,
"distanceThreshold": distanceThreshold,
"conservativeDistanceThreshold": distanceThreshold,
"useDynamicThreshold": false,
},
taskName: "createImageEmbedding",
) as ClusteringResult;
final endTime = DateTime.now();
_logger.info(
"Linear Clustering took: ${endTime.difference(startTime).inMilliseconds}ms",
);
return faceIdToCluster;
} catch (e, s) {
_logger.severe(e, s);
rethrow;
}
}
/// Runs the clustering algorithm [runCompleteClustering] on the given [input], in computer.
///
/// WARNING: Only use on small datasets, as it is not optimized for large datasets.
Future<Map<String, int>> predictCompleteComputer(
Map<String, Uint8List> input, {
Map<int, int>? fileIDToCreationTime,
double distanceThreshold = kRecommendedDistanceThreshold,
double mergeThreshold = 0.30,
}) async {
if (input.isEmpty) {
_logger.warning(
"Complete Clustering dataset of embeddings is empty, returning empty list.",
);
return {};
}
// Clustering inside the isolate
_logger.info(
"Start Complete clustering on ${input.length} embeddings inside computer isolate",
);
try {
final startTime = DateTime.now();
final faceIdToCluster = await _computer.compute(
runCompleteClustering,
param: {
"input": input,
"fileIDToCreationTime": fileIDToCreationTime,
"distanceThreshold": distanceThreshold,
"mergeThreshold": mergeThreshold,
},
taskName: "createImageEmbedding",
) as Map<String, int>;
final endTime = DateTime.now();
_logger.info(
"Complete Clustering took: ${endTime.difference(startTime).inMilliseconds}ms",
);
return faceIdToCluster;
} catch (e, s) {
_logger.severe(e, s);
rethrow;
}
}
Future<Map<String, int>?> predictWithinClusterComputer(
Map<String, Uint8List> input, {
Map<int, int>? fileIDToCreationTime,
double distanceThreshold = kRecommendedDistanceThreshold,
}) async {
_logger.info(
'`predictWithinClusterComputer` called with ${input.length} faces and distance threshold $distanceThreshold',
);
try {
if (input.length < 100) {
final mergeThreshold = distanceThreshold + 0.06;
_logger.info(
'Running complete clustering on ${input.length} faces with distance threshold $mergeThreshold',
);
return predictCompleteComputer(
input,
fileIDToCreationTime: fileIDToCreationTime,
mergeThreshold: mergeThreshold,
);
} else {
_logger.info(
'Running linear clustering on ${input.length} faces with distance threshold $distanceThreshold',
);
final clusterResult = await predictLinearComputer(
input,
fileIDToCreationTime: fileIDToCreationTime,
distanceThreshold: distanceThreshold,
);
return clusterResult?.newFaceIdToCluster;
}
} catch (e, s) {
_logger.severe(e, s);
rethrow;
}
}
Future<List<List<String>>> predictDbscan(
Map<String, Uint8List> input, {
Map<int, int>? fileIDToCreationTime,
@ -299,29 +442,42 @@ class FaceClusteringService {
return clusterFaceIDs;
}
static Map<String, int> _runLinearClustering(
Map<String, (int?, Uint8List)> x, {
Map<int, int>? fileIDToCreationTime,
double distanceThreshold = kRecommendedDistanceThreshold,
int? offset,
}) {
static ClusteringResult? runLinearClustering(Map args) {
// final input = args['input'] as Map<String, (int?, Uint8List)>;
final input = args['input'] as Set<FaceInfoForClustering>;
final fileIDToCreationTime = args['fileIDToCreationTime'] as Map<int, int>?;
final distanceThreshold = args['distanceThreshold'] as double;
final conservativeDistanceThreshold =
args['conservativeDistanceThreshold'] as double;
final useDynamicThreshold = args['useDynamicThreshold'] as bool;
final offset = args['offset'] as int?;
final oldClusterSummaries =
args['oldClusterSummaries'] as Map<int, (Uint8List, int)>?;
log(
"[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces",
"[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces",
);
// Organize everything into a list of FaceInfo objects
final List<FaceInfo> faceInfos = [];
for (final entry in x.entries) {
for (final face in input) {
faceInfos.add(
FaceInfo(
faceID: entry.key,
faceID: face.faceID,
faceScore: face.faceScore,
blurValue: face.blurValue,
badFace: face.faceScore < kMinimumQualityFaceScore ||
face.blurValue < kLaplacianSoftThreshold ||
(face.blurValue < kLaplacianVerySoftThreshold &&
face.faceScore < kMediumQualityFaceScore) ||
face.isSideways,
vEmbedding: Vector.fromList(
EVector.fromBuffer(entry.value.$2).values,
EVector.fromBuffer(face.embeddingBytes).values,
dtype: DType.float32,
),
clusterId: entry.value.$1,
clusterId: face.clusterId,
fileCreationTime:
fileIDToCreationTime?[getFileIdFromFaceId(entry.key)],
fileIDToCreationTime?[getFileIdFromFaceId(face.faceID)],
),
);
}
@ -351,19 +507,21 @@ class FaceClusteringService {
facesWithClusterID.add(faceInfo);
}
}
final alreadyClusteredCount = facesWithClusterID.length;
final sortedFaceInfos = <FaceInfo>[];
sortedFaceInfos.addAll(facesWithClusterID);
sortedFaceInfos.addAll(facesWithoutClusterID);
log(
"[ClusterIsolate] ${DateTime.now()} Clustering ${facesWithoutClusterID.length} new faces without clusterId, and ${facesWithClusterID.length} faces with clusterId",
"[ClusterIsolate] ${DateTime.now()} Clustering ${facesWithoutClusterID.length} new faces without clusterId, and $alreadyClusteredCount faces with clusterId",
);
// Make sure the first face has a clusterId
final int totalFaces = sortedFaceInfos.length;
int dynamicThresholdCount = 0;
if (sortedFaceInfos.isEmpty) {
return {};
return null;
}
// Start actual clustering
@ -377,7 +535,6 @@ class FaceClusteringService {
sortedFaceInfos[0].clusterId = clusterID;
clusterID++;
}
final Map<String, int> newFaceIdToCluster = {};
final stopwatchClustering = Stopwatch()..start();
for (int i = 1; i < totalFaces; i++) {
// Incremental clustering, so we can skip faces that already have a clusterId
@ -388,6 +545,15 @@ class FaceClusteringService {
int closestIdx = -1;
double closestDistance = double.infinity;
late double thresholdValue;
if (useDynamicThreshold) {
thresholdValue = sortedFaceInfos[i].badFace!
? conservativeDistanceThreshold
: distanceThreshold;
if (sortedFaceInfos[i].badFace!) dynamicThresholdCount++;
} else {
thresholdValue = distanceThreshold;
}
if (i % 250 == 0) {
log("[ClusterIsolate] ${DateTime.now()} Processed ${offset != null ? i + offset : i} faces");
}
@ -405,18 +571,16 @@ class FaceClusteringService {
);
}
if (distance < closestDistance) {
if (sortedFaceInfos[j].badFace! &&
distance > conservativeDistanceThreshold) {
continue;
}
closestDistance = distance;
closestIdx = j;
// if (distance < distanceThreshold) {
// if (sortedFaceInfos[j].faceID.startsWith("14914702") ||
// sortedFaceInfos[j].faceID.startsWith("15488756")) {
// log('[XXX] faceIDs: ${sortedFaceInfos[j].faceID} and ${sortedFaceInfos[i].faceID} with distance $distance');
// }
// }
}
}
if (closestDistance < distanceThreshold) {
if (closestDistance < thresholdValue) {
if (sortedFaceInfos[closestIdx].clusterId == null) {
// Ideally this should never happen, but just in case log it
log(
@ -424,42 +588,99 @@ class FaceClusteringService {
);
clusterID++;
sortedFaceInfos[closestIdx].clusterId = clusterID;
newFaceIdToCluster[sortedFaceInfos[closestIdx].faceID] = clusterID;
}
// if (sortedFaceInfos[i].faceID.startsWith("14914702") ||
// sortedFaceInfos[i].faceID.startsWith("15488756")) {
// log(
// "[XXX] [ClusterIsolate] ${DateTime.now()} Found similar face ${sortedFaceInfos[i].faceID} to ${sortedFaceInfos[closestIdx].faceID} with distance $closestDistance",
// );
// }
sortedFaceInfos[i].clusterId = sortedFaceInfos[closestIdx].clusterId;
newFaceIdToCluster[sortedFaceInfos[i].faceID] =
sortedFaceInfos[closestIdx].clusterId!;
} else {
// if (sortedFaceInfos[i].faceID.startsWith("14914702") ||
// sortedFaceInfos[i].faceID.startsWith("15488756")) {
// log(
// "[XXX] [ClusterIsolate] ${DateTime.now()} Found new cluster $clusterID for face ${sortedFaceInfos[i].faceID}",
// );
// }
clusterID++;
sortedFaceInfos[i].clusterId = clusterID;
newFaceIdToCluster[sortedFaceInfos[i].faceID] = clusterID;
}
}
// Finally, assign the new clusterId to the faces
final Map<String, int> newFaceIdToCluster = {};
final newClusteredFaceInfos =
sortedFaceInfos.sublist(alreadyClusteredCount);
for (final faceInfo in newClusteredFaceInfos) {
newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!;
}
stopwatchClustering.stop();
log(
' [ClusterIsolate] ${DateTime.now()} Clustering for ${sortedFaceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms',
);
if (useDynamicThreshold) {
log(
"[ClusterIsolate] ${DateTime.now()} Dynamic thresholding: $dynamicThresholdCount faces had a low face score or low blur clarity",
);
}
// Now calculate the mean of the embeddings for each cluster and update the cluster summaries
Map<int, (Uint8List, int)>? newClusterSummaries;
if (oldClusterSummaries != null) {
newClusterSummaries = FaceClusteringService.updateClusterSummaries(
oldSummary: oldClusterSummaries,
newFaceInfos: newClusteredFaceInfos,
);
}
// analyze the results
FaceClusteringService._analyzeClusterResults(sortedFaceInfos);
return newFaceIdToCluster;
return ClusteringResult(
newFaceIdToCluster: newFaceIdToCluster,
newClusterSummaries: newClusterSummaries,
);
}
static Map<int, (Uint8List, int)> updateClusterSummaries({
required Map<int, (Uint8List, int)> oldSummary,
required List<FaceInfo> newFaceInfos,
}) {
final calcSummariesStart = DateTime.now();
final Map<int, List<FaceInfo>> newClusterIdToFaceInfos = {};
for (final faceInfo in newFaceInfos) {
if (newClusterIdToFaceInfos.containsKey(faceInfo.clusterId!)) {
newClusterIdToFaceInfos[faceInfo.clusterId!]!.add(faceInfo);
} else {
newClusterIdToFaceInfos[faceInfo.clusterId!] = [faceInfo];
}
}
final Map<int, (Uint8List, int)> newClusterSummaries = {};
for (final clusterId in newClusterIdToFaceInfos.keys) {
final List<Vector> newEmbeddings = newClusterIdToFaceInfos[clusterId]!
.map((faceInfo) => faceInfo.vEmbedding!)
.toList();
final newCount = newEmbeddings.length;
if (oldSummary.containsKey(clusterId)) {
final oldMean = Vector.fromList(
EVector.fromBuffer(oldSummary[clusterId]!.$1).values,
dtype: DType.float32,
);
final oldCount = oldSummary[clusterId]!.$2;
final oldEmbeddings = oldMean * oldCount;
newEmbeddings.add(oldEmbeddings);
final newMeanVector =
newEmbeddings.reduce((a, b) => a + b) / (oldCount + newCount);
newClusterSummaries[clusterId] = (
EVector(values: newMeanVector.toList()).writeToBuffer(),
oldCount + newCount
);
} else {
final newMeanVector = newEmbeddings.reduce((a, b) => a + b) / newCount;
newClusterSummaries[clusterId] =
(EVector(values: newMeanVector.toList()).writeToBuffer(), newCount);
}
}
log(
"[ClusterIsolate] ${DateTime.now()} Calculated cluster summaries in ${DateTime.now().difference(calcSummariesStart).inMilliseconds}ms",
);
return newClusterSummaries;
}
static void _analyzeClusterResults(List<FaceInfo> sortedFaceInfos) {
if (!kDebugMode) return;
final stopwatch = Stopwatch()..start();
final Map<String, int> faceIdToCluster = {};
@ -517,14 +738,185 @@ class FaceClusteringService {
);
}
static List<List<String>> _runDbscanClustering(
Map<String, Uint8List> x, {
Map<int, int>? fileIDToCreationTime,
double eps = 0.3,
int minPts = 5,
}) {
static Map<String, int> runCompleteClustering(Map args) {
final input = args['input'] as Map<String, Uint8List>;
final fileIDToCreationTime = args['fileIDToCreationTime'] as Map<int, int>?;
final distanceThreshold = args['distanceThreshold'] as double;
final mergeThreshold = args['mergeThreshold'] as double;
log(
"[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces",
"[CompleteClustering] ${DateTime.now()} Copied to isolate ${input.length} faces for clustering",
);
// Organize everything into a list of FaceInfo objects
final List<FaceInfo> faceInfos = [];
for (final entry in input.entries) {
faceInfos.add(
FaceInfo(
faceID: entry.key,
vEmbedding: Vector.fromList(
EVector.fromBuffer(entry.value).values,
dtype: DType.float32,
),
fileCreationTime:
fileIDToCreationTime?[getFileIdFromFaceId(entry.key)],
),
);
}
// Sort the faceInfos based on fileCreationTime, in ascending order, so oldest faces are first
if (fileIDToCreationTime != null) {
faceInfos.sort((a, b) {
if (a.fileCreationTime == null && b.fileCreationTime == null) {
return 0;
} else if (a.fileCreationTime == null) {
return 1;
} else if (b.fileCreationTime == null) {
return -1;
} else {
return a.fileCreationTime!.compareTo(b.fileCreationTime!);
}
});
}
if (faceInfos.isEmpty) {
return {};
}
final int totalFaces = faceInfos.length;
// Start actual clustering
log(
"[CompleteClustering] ${DateTime.now()} Processing $totalFaces faces in one single round of complete clustering",
);
// set current epoch time as clusterID
int clusterID = DateTime.now().microsecondsSinceEpoch;
// Start actual clustering
final Map<String, int> newFaceIdToCluster = {};
final stopwatchClustering = Stopwatch()..start();
for (int i = 0; i < totalFaces; i++) {
if ((i + 1) % 250 == 0) {
log("[CompleteClustering] ${DateTime.now()} Processed ${i + 1} faces");
}
if (faceInfos[i].clusterId != null) continue;
int closestIdx = -1;
double closestDistance = double.infinity;
for (int j = 0; j < totalFaces; j++) {
if (i == j) continue;
final double distance =
1.0 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!);
if (distance < closestDistance) {
closestDistance = distance;
closestIdx = j;
}
}
if (closestDistance < distanceThreshold) {
if (faceInfos[closestIdx].clusterId == null) {
clusterID++;
faceInfos[closestIdx].clusterId = clusterID;
}
faceInfos[i].clusterId = faceInfos[closestIdx].clusterId!;
} else {
clusterID++;
faceInfos[i].clusterId = clusterID;
}
}
// Now calculate the mean of the embeddings for each cluster
final Map<int, List<FaceInfo>> clusterIdToFaceInfos = {};
for (final faceInfo in faceInfos) {
if (clusterIdToFaceInfos.containsKey(faceInfo.clusterId)) {
clusterIdToFaceInfos[faceInfo.clusterId]!.add(faceInfo);
} else {
clusterIdToFaceInfos[faceInfo.clusterId!] = [faceInfo];
}
}
final Map<int, (Vector, int)> clusterIdToMeanEmbeddingAndWeight = {};
for (final clusterId in clusterIdToFaceInfos.keys) {
final List<Vector> embeddings = clusterIdToFaceInfos[clusterId]!
.map((faceInfo) => faceInfo.vEmbedding!)
.toList();
final count = clusterIdToFaceInfos[clusterId]!.length;
final Vector meanEmbedding = embeddings.reduce((a, b) => a + b) / count;
clusterIdToMeanEmbeddingAndWeight[clusterId] = (meanEmbedding, count);
}
// Now merge the clusters that are close to each other, based on mean embedding
final List<(int, int)> mergedClustersList = [];
final List<int> clusterIds =
clusterIdToMeanEmbeddingAndWeight.keys.toList();
log(' [CompleteClustering] ${DateTime.now()} ${clusterIds.length} clusters found, now checking for merges');
while (true) {
if (clusterIds.length < 2) break;
double distance = double.infinity;
(int, int) clusterIDsToMerge = (-1, -1);
for (int i = 0; i < clusterIds.length; i++) {
for (int j = 0; j < clusterIds.length; j++) {
if (i == j) continue;
final double newDistance = 1.0 -
clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1.dot(
clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1,
);
if (newDistance < distance) {
distance = newDistance;
clusterIDsToMerge = (clusterIds[i], clusterIds[j]);
}
}
}
if (distance < mergeThreshold) {
mergedClustersList.add(clusterIDsToMerge);
final clusterID1 = clusterIDsToMerge.$1;
final clusterID2 = clusterIDsToMerge.$2;
final mean1 = clusterIdToMeanEmbeddingAndWeight[clusterID1]!.$1;
final mean2 = clusterIdToMeanEmbeddingAndWeight[clusterID2]!.$1;
final count1 = clusterIdToMeanEmbeddingAndWeight[clusterID1]!.$2;
final count2 = clusterIdToMeanEmbeddingAndWeight[clusterID2]!.$2;
final weight1 = count1 / (count1 + count2);
final weight2 = count2 / (count1 + count2);
clusterIdToMeanEmbeddingAndWeight[clusterID1] = (
mean1 * weight1 + mean2 * weight2,
count1 + count2,
);
clusterIdToMeanEmbeddingAndWeight.remove(clusterID2);
clusterIds.remove(clusterID2);
} else {
break;
}
}
log(' [CompleteClustering] ${DateTime.now()} ${mergedClustersList.length} clusters merged');
// Now assign the new clusterId to the faces
for (final faceInfo in faceInfos) {
for (final mergedClusters in mergedClustersList) {
if (faceInfo.clusterId == mergedClusters.$2) {
faceInfo.clusterId = mergedClusters.$1;
}
}
}
// Finally, assign the new clusterId to the faces
for (final faceInfo in faceInfos) {
newFaceIdToCluster[faceInfo.faceID] = faceInfo.clusterId!;
}
stopwatchClustering.stop();
log(
' [CompleteClustering] ${DateTime.now()} Clustering for ${faceInfos.length} embeddings executed in ${stopwatchClustering.elapsedMilliseconds}ms',
);
return newFaceIdToCluster;
}
static List<List<String>> _runDbscanClustering(Map args) {
final input = args['input'] as Map<String, Uint8List>;
final fileIDToCreationTime = args['fileIDToCreationTime'] as Map<int, int>?;
final eps = args['eps'] as double;
final minPts = args['minPts'] as int;
log(
"[ClusterIsolate] ${DateTime.now()} Copied to isolate ${input.length} faces",
);
final DBSCAN dbscan = DBSCAN(
@ -535,7 +927,7 @@ class FaceClusteringService {
// Organize everything into a list of FaceInfo objects
final List<FaceInfo> faceInfos = [];
for (final entry in x.entries) {
for (final entry in input.entries) {
faceInfos.add(
FaceInfo(
faceID: entry.key,

View file

@ -0,0 +1,19 @@
import "dart:typed_data" show Uint8List;
class FaceInfoForClustering {
final String faceID;
final int? clusterId;
final Uint8List embeddingBytes;
final double faceScore;
final double blurValue;
final bool isSideways;
FaceInfoForClustering({
required this.faceID,
this.clusterId,
required this.embeddingBytes,
required this.faceScore,
required this.blurValue,
this.isSideways = false,
});
}

View file

@ -1,7 +1,24 @@
import 'dart:math' show sqrt, pow;
import 'dart:math' show max, min, pow, sqrt;
import "package:photos/face/model/dimension.dart";
enum FaceDirection { left, right, straight }
extension FaceDirectionExtension on FaceDirection {
String toDirectionString() {
switch (this) {
case FaceDirection.left:
return 'Left';
case FaceDirection.right:
return 'Right';
case FaceDirection.straight:
return 'Straight';
default:
throw Exception('Unknown FaceDirection');
}
}
}
abstract class Detection {
final double score;
@ -16,6 +33,7 @@ abstract class Detection {
String toString();
}
@Deprecated('Old method only used in other deprecated methods')
extension BBoxExtension on List<double> {
void roundBoxToDouble() {
final widthRounded = (this[2] - this[0]).roundToDouble();
@ -425,6 +443,37 @@ class FaceDetectionAbsolute extends Detection {
/// The height of the bounding box of the face detection, in number of pixels, range [0, imageHeight].
double get height => yMaxBox - yMinBox;
FaceDirection getFaceDirection() {
final double eyeDistanceX = (rightEye[0] - leftEye[0]).abs();
final double eyeDistanceY = (rightEye[1] - leftEye[1]).abs();
final double mouthDistanceY = (rightMouth[1] - leftMouth[1]).abs();
final bool faceIsUpright =
(max(leftEye[1], rightEye[1]) + 0.5 * eyeDistanceY < nose[1]) &&
(nose[1] + 0.5 * mouthDistanceY < min(leftMouth[1], rightMouth[1]));
final bool noseStickingOutLeft = (nose[0] < min(leftEye[0], rightEye[0])) &&
(nose[0] < min(leftMouth[0], rightMouth[0]));
final bool noseStickingOutRight =
(nose[0] > max(leftEye[0], rightEye[0])) &&
(nose[0] > max(leftMouth[0], rightMouth[0]));
final bool noseCloseToLeftEye =
(nose[0] - leftEye[0]).abs() < 0.2 * eyeDistanceX;
final bool noseCloseToRightEye =
(nose[0] - rightEye[0]).abs() < 0.2 * eyeDistanceX;
// if (faceIsUpright && (noseStickingOutLeft || noseCloseToLeftEye)) {
if (noseStickingOutLeft || (faceIsUpright && noseCloseToLeftEye)) {
return FaceDirection.left;
// } else if (faceIsUpright && (noseStickingOutRight || noseCloseToRightEye)) {
} else if (noseStickingOutRight || (faceIsUpright && noseCloseToRightEye)) {
return FaceDirection.right;
}
return FaceDirection.straight;
}
}
List<FaceDetectionAbsolute> relativeToAbsoluteDetections({

View file

@ -1,4 +1,5 @@
import 'package:logging/logging.dart';
import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
class BlurDetectionService {
@ -11,9 +12,11 @@ class BlurDetectionService {
Future<(bool, double)> predictIsBlurGrayLaplacian(
List<List<int>> grayImage, {
int threshold = kLaplacianThreshold,
int threshold = kLaplacianHardThreshold,
FaceDirection faceDirection = FaceDirection.straight,
}) async {
final List<List<int>> laplacian = _applyLaplacian(grayImage);
final List<List<int>> laplacian =
_applyLaplacian(grayImage, faceDirection: faceDirection);
final double variance = _calculateVariance(laplacian);
_logger.info('Variance: $variance');
return (variance < threshold, variance);
@ -46,43 +49,80 @@ class BlurDetectionService {
return variance;
}
List<List<int>> _padImage(List<List<int>> image) {
List<List<int>> _padImage(
List<List<int>> image, {
int removeSideColumns = 56,
FaceDirection faceDirection = FaceDirection.straight,
}) {
// Exception is removeSideColumns is not even
if (removeSideColumns % 2 != 0) {
throw Exception('removeSideColumns must be even');
}
final int numRows = image.length;
final int numCols = image[0].length;
final int paddedNumCols = numCols + 2 - removeSideColumns;
final int paddedNumRows = numRows + 2;
// Create a new matrix with extra padding
final List<List<int>> paddedImage = List.generate(
numRows + 2,
(i) => List.generate(numCols + 2, (j) => 0, growable: false),
paddedNumRows,
(i) => List.generate(
paddedNumCols,
(j) => 0,
growable: false,
),
growable: false,
);
// Copy original image into the center of the padded image
for (int i = 0; i < numRows; i++) {
for (int j = 0; j < numCols; j++) {
paddedImage[i + 1][j + 1] = image[i][j];
// Copy original image into the center of the padded image, taking into account the face direction
if (faceDirection == FaceDirection.straight) {
for (int i = 0; i < numRows; i++) {
for (int j = 0; j < (paddedNumCols - 2); j++) {
paddedImage[i + 1][j + 1] =
image[i][j + (removeSideColumns / 2).round()];
}
}
// If the face is facing left, we only take the right side of the face image
} else if (faceDirection == FaceDirection.left) {
for (int i = 0; i < numRows; i++) {
for (int j = 0; j < (paddedNumCols - 2); j++) {
paddedImage[i + 1][j + 1] = image[i][j + removeSideColumns];
}
}
// If the face is facing right, we only take the left side of the face image
} else if (faceDirection == FaceDirection.right) {
for (int i = 0; i < numRows; i++) {
for (int j = 0; j < (paddedNumCols - 2); j++) {
paddedImage[i + 1][j + 1] = image[i][j];
}
}
}
// Reflect padding
// Top and bottom rows
for (int j = 1; j <= numCols; j++) {
for (int j = 1; j <= (paddedNumCols - 2); j++) {
paddedImage[0][j] = paddedImage[2][j]; // Top row
paddedImage[numRows + 1][j] = paddedImage[numRows - 1][j]; // Bottom row
}
// Left and right columns
for (int i = 0; i < numRows + 2; i++) {
paddedImage[i][0] = paddedImage[i][2]; // Left column
paddedImage[i][numCols + 1] = paddedImage[i][numCols - 1]; // Right column
paddedImage[i][paddedNumCols - 1] =
paddedImage[i][paddedNumCols - 3]; // Right column
}
return paddedImage;
}
List<List<int>> _applyLaplacian(List<List<int>> image) {
final List<List<int>> paddedImage = _padImage(image);
final int numRows = image.length;
final int numCols = image[0].length;
List<List<int>> _applyLaplacian(
List<List<int>> image, {
FaceDirection faceDirection = FaceDirection.straight,
}) {
final List<List<int>> paddedImage =
_padImage(image, faceDirection: faceDirection);
final int numRows = paddedImage.length - 2;
final int numCols = paddedImage[0].length - 2;
final List<List<int>> outputImage = List.generate(
numRows,
(i) => List.generate(numCols, (j) => 0, growable: false),

View file

@ -1,13 +1,17 @@
import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart';
/// Blur detection threshold
const kLaplacianThreshold = 15;
const kLaplacianHardThreshold = 15;
const kLaplacianSoftThreshold = 100;
const kLaplacianVerySoftThreshold = 200;
/// Default blur value
const kLapacianDefault = 10000.0;
/// The minimum score for a face to be considered a high quality face for clustering and person detection
const kMinHighQualityFaceScore = 0.80;
const kMinimumQualityFaceScore = 0.80;
const kMediumQualityFaceScore = 0.85;
const kHighQualityFaceScore = 0.90;
/// The minimum score for a face to be detected, regardless of quality. Use [kMinHighQualityFaceScore] for high quality faces.
/// The minimum score for a face to be detected, regardless of quality. Use [kMinimumQualityFaceScore] for high quality faces.
const kMinFaceDetectionScore = FaceDetectionService.kMinScoreSigmoidThreshold;

View file

@ -1,284 +1,18 @@
import "dart:convert" show jsonEncode, jsonDecode;
import "package:flutter/material.dart" show debugPrint, immutable;
import "package:flutter/material.dart" show immutable;
import "package:logging/logging.dart";
import "package:photos/face/model/dimension.dart";
import "package:photos/models/file/file.dart";
import 'package:photos/models/ml/ml_typedefs.dart';
import "package:photos/models/ml/ml_versions.dart";
import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart';
import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart';
import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart';
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
import 'package:photos/services/machine_learning/face_ml/face_ml_methods.dart';
final _logger = Logger('ClusterResult_FaceMlResult');
// TODO: should I add [faceMlVersion] and [clusterMlVersion] to the [ClusterResult] class?
@Deprecated('We are now just storing the cluster results directly in DB')
class ClusterResult {
final int personId;
String? userDefinedName;
bool get hasUserDefinedName => userDefinedName != null;
String _thumbnailFaceId;
bool thumbnailFaceIdIsUserDefined;
final List<int> _fileIds;
final List<String> _faceIds;
final Embedding medoid;
double medoidDistanceThreshold;
List<int> get uniqueFileIds => _fileIds.toSet().toList();
List<int> get fileIDsIncludingPotentialDuplicates => _fileIds;
List<String> get faceIDs => _faceIds;
String get thumbnailFaceId => _thumbnailFaceId;
int get thumbnailFileId => getFileIdFromFaceId(_thumbnailFaceId);
/// Sets the thumbnail faceId to the given faceId.
/// Throws an exception if the faceId is not in the list of faceIds.
set setThumbnailFaceId(String faceId) {
if (!_faceIds.contains(faceId)) {
throw Exception(
"The faceId $faceId is not in the list of faceIds: $faceId",
);
}
_thumbnailFaceId = faceId;
thumbnailFaceIdIsUserDefined = true;
}
/// Sets the [userDefinedName] to the given [customName]
set setUserDefinedName(String customName) {
userDefinedName = customName;
}
int get clusterSize => _fileIds.toSet().length;
ClusterResult({
required this.personId,
required String thumbnailFaceId,
required List<int> fileIds,
required List<String> faceIds,
required this.medoid,
required this.medoidDistanceThreshold,
this.userDefinedName,
this.thumbnailFaceIdIsUserDefined = false,
}) : _thumbnailFaceId = thumbnailFaceId,
_faceIds = faceIds,
_fileIds = fileIds;
void addFileIDsAndFaceIDs(List<int> fileIDs, List<String> faceIDs) {
assert(fileIDs.length == faceIDs.length);
_fileIds.addAll(fileIDs);
_faceIds.addAll(faceIDs);
}
// TODO: Consider if we should recalculated the medoid and threshold when deleting or adding a file from the cluster
int removeFileId(int fileId) {
assert(_fileIds.length == _faceIds.length);
if (!_fileIds.contains(fileId)) {
throw Exception(
"The fileId $fileId is not in the list of fileIds: $fileId, so it's not in the cluster and cannot be removed.",
);
}
int removedCount = 0;
for (var i = 0; i < _fileIds.length; i++) {
if (_fileIds[i] == fileId) {
assert(getFileIdFromFaceId(_faceIds[i]) == fileId);
_fileIds.removeAt(i);
_faceIds.removeAt(i);
debugPrint(
"Removed fileId $fileId from cluster $personId at index ${i + removedCount}}",
);
i--; // Adjust index due to removal
removedCount++;
}
}
_ensureClusterSizeIsAboveMinimum();
return removedCount;
}
int addFileID(int fileID) {
assert(_fileIds.length == _faceIds.length);
if (_fileIds.contains(fileID)) {
return 0;
}
_fileIds.add(fileID);
_faceIds.add(FaceDetectionRelative.toFaceIDEmpty(fileID: fileID));
return 1;
}
void ensureThumbnailFaceIdIsInCluster() {
if (!_faceIds.contains(_thumbnailFaceId)) {
_thumbnailFaceId = _faceIds[0];
}
}
void _ensureClusterSizeIsAboveMinimum() {
if (clusterSize < minimumClusterSize) {
throw Exception(
"Cluster size is below minimum cluster size of $minimumClusterSize",
);
}
}
Map<String, dynamic> _toJson() => {
'personId': personId,
'thumbnailFaceId': _thumbnailFaceId,
'fileIds': _fileIds,
'faceIds': _faceIds,
'medoid': medoid,
'medoidDistanceThreshold': medoidDistanceThreshold,
if (userDefinedName != null) 'userDefinedName': userDefinedName,
'thumbnailFaceIdIsUserDefined': thumbnailFaceIdIsUserDefined,
};
String toJsonString() => jsonEncode(_toJson());
static ClusterResult _fromJson(Map<String, dynamic> json) {
return ClusterResult(
personId: json['personId'] ?? -1,
thumbnailFaceId: json['thumbnailFaceId'] ?? '',
fileIds:
(json['fileIds'] as List?)?.map((item) => item as int).toList() ?? [],
faceIds:
(json['faceIds'] as List?)?.map((item) => item as String).toList() ??
[],
medoid:
(json['medoid'] as List?)?.map((item) => item as double).toList() ??
[],
medoidDistanceThreshold: json['medoidDistanceThreshold'] ?? 0,
userDefinedName: json['userDefinedName'],
thumbnailFaceIdIsUserDefined:
json['thumbnailFaceIdIsUserDefined'] as bool,
);
}
static ClusterResult fromJsonString(String jsonString) {
return _fromJson(jsonDecode(jsonString));
}
}
class ClusterResultBuilder {
int personId = -1;
String? userDefinedName;
String thumbnailFaceId = '';
bool thumbnailFaceIdIsUserDefined = false;
List<int> fileIds = <int>[];
List<String> faceIds = <String>[];
List<Embedding> embeddings = <Embedding>[];
Embedding medoid = <double>[];
double medoidDistanceThreshold = 0;
bool medoidAndThresholdCalculated = false;
final int k = 5;
ClusterResultBuilder.createFromIndices({
required List<int> clusterIndices,
required List<int> labels,
required List<Embedding> allEmbeddings,
required List<int> allFileIds,
required List<String> allFaceIds,
}) {
final clusteredFileIds =
clusterIndices.map((fileIndex) => allFileIds[fileIndex]).toList();
final clusteredFaceIds =
clusterIndices.map((fileIndex) => allFaceIds[fileIndex]).toList();
final clusteredEmbeddings =
clusterIndices.map((fileIndex) => allEmbeddings[fileIndex]).toList();
personId = labels[clusterIndices[0]];
fileIds = clusteredFileIds;
faceIds = clusteredFaceIds;
thumbnailFaceId = faceIds[0];
embeddings = clusteredEmbeddings;
}
void calculateAndSetMedoidAndThreshold() {
if (embeddings.isEmpty) {
throw Exception("Cannot calculate medoid and threshold for empty list");
}
// Calculate the medoid and threshold
final (tempMedoid, distanceThreshold) =
_calculateMedoidAndDistanceTreshold(embeddings);
// Update the medoid
medoid = List.from(tempMedoid);
// Update the medoidDistanceThreshold as the distance of the medoid to its k-th nearest neighbor
medoidDistanceThreshold = distanceThreshold;
medoidAndThresholdCalculated = true;
}
(List<double>, double) _calculateMedoidAndDistanceTreshold(
List<List<double>> embeddings,
) {
double minDistance = double.infinity;
List<double>? medoid;
// Calculate the distance between all pairs
for (int i = 0; i < embeddings.length; ++i) {
double totalDistance = 0;
for (int j = 0; j < embeddings.length; ++j) {
if (i != j) {
totalDistance += cosineDistance(embeddings[i], embeddings[j]);
// Break early if we already exceed minDistance
if (totalDistance > minDistance) {
break;
}
}
}
// Find the minimum total distance
if (totalDistance < minDistance) {
minDistance = totalDistance;
medoid = embeddings[i];
}
}
// Now, calculate k-th nearest neighbor for the medoid
final List<double> distancesToMedoid = [];
for (List<double> embedding in embeddings) {
if (embedding != medoid) {
distancesToMedoid.add(cosineDistance(medoid!, embedding));
}
}
distancesToMedoid.sort();
// TODO: empirically find the best k. Probably it should be dynamic in some way, so for instance larger for larger clusters and smaller for smaller clusters, especially since there are a lot of really small clusters and a few really large ones.
final double kthDistance = distancesToMedoid[
distancesToMedoid.length >= k ? k - 1 : distancesToMedoid.length - 1];
return (medoid!, kthDistance);
}
void changeThumbnailFaceId(String faceId) {
if (!faceIds.contains(faceId)) {
throw Exception(
"The faceId $faceId is not in the list of faceIds: $faceIds",
);
}
thumbnailFaceId = faceId;
}
void addFileIDsAndFaceIDs(List<int> addedFileIDs, List<String> addedFaceIDs) {
assert(addedFileIDs.length == addedFaceIDs.length);
fileIds.addAll(addedFileIDs);
faceIds.addAll(addedFaceIDs);
}
}
@immutable
class FaceMlResult {
final int fileId;
@ -504,7 +238,7 @@ class FaceResult {
final int fileId;
final String faceId;
bool get isBlurry => blurValue < kLaplacianThreshold;
bool get isBlurry => blurValue < kLaplacianHardThreshold;
const FaceResult({
required this.detection,
@ -545,7 +279,7 @@ class FaceResultBuilder {
int fileId = -1;
String faceId = '';
bool get isBlurry => blurValue < kLaplacianThreshold;
bool get isBlurry => blurValue < kLaplacianHardThreshold;
FaceResultBuilder({
required this.fileId,

View file

@ -204,83 +204,13 @@ class FaceMlService {
try {
switch (function) {
case FaceMlOperation.analyzeImage:
final int enteFileID = args["enteFileID"] as int;
final String imagePath = args["filePath"] as String;
final int faceDetectionAddress =
args["faceDetectionAddress"] as int;
final int faceEmbeddingAddress =
args["faceEmbeddingAddress"] as int;
final resultBuilder =
FaceMlResultBuilder.fromEnteFileID(enteFileID);
final time = DateTime.now();
final FaceMlResult result =
await FaceMlService.analyzeImageSync(args);
dev.log(
"Start analyzing image with uploadedFileID: $enteFileID inside the isolate",
"`analyzeImageSync` function executed in ${DateTime.now().difference(time).inMilliseconds} ms",
);
final stopwatchTotal = Stopwatch()..start();
final stopwatch = Stopwatch()..start();
// Decode the image once to use for both face detection and alignment
final imageData = await File(imagePath).readAsBytes();
final image = await decodeImageFromData(imageData);
final ByteData imgByteData = await getByteDataFromImage(image);
dev.log('Reading and decoding image took '
'${stopwatch.elapsedMilliseconds} ms');
stopwatch.reset();
// Get the faces
final List<FaceDetectionRelative> faceDetectionResult =
await FaceMlService.detectFacesSync(
image,
imgByteData,
faceDetectionAddress,
resultBuilder: resultBuilder,
);
dev.log(
"${faceDetectionResult.length} faces detected with scores ${faceDetectionResult.map((e) => e.score).toList()}: completed `detectFacesSync` function, in "
"${stopwatch.elapsedMilliseconds} ms");
// If no faces were detected, return a result with no faces. Otherwise, continue.
if (faceDetectionResult.isEmpty) {
dev.log(
"No faceDetectionResult, Completed analyzing image with uploadedFileID $enteFileID, in "
"${stopwatch.elapsedMilliseconds} ms");
sendPort.send(resultBuilder.buildNoFaceDetected().toJsonString());
break;
}
stopwatch.reset();
// Align the faces
final Float32List faceAlignmentResult =
await FaceMlService.alignFacesSync(
image,
imgByteData,
faceDetectionResult,
resultBuilder: resultBuilder,
);
dev.log("Completed `alignFacesSync` function, in "
"${stopwatch.elapsedMilliseconds} ms");
stopwatch.reset();
// Get the embeddings of the faces
final embeddings = await FaceMlService.embedFacesSync(
faceAlignmentResult,
faceEmbeddingAddress,
resultBuilder: resultBuilder,
);
dev.log("Completed `embedFacesSync` function, in "
"${stopwatch.elapsedMilliseconds} ms");
stopwatch.stop();
stopwatchTotal.stop();
dev.log("Finished Analyze image (${embeddings.length} faces) with "
"uploadedFileID $enteFileID, in "
"${stopwatchTotal.elapsedMilliseconds} ms");
sendPort.send(resultBuilder.build().toJsonString());
sendPort.send(result.toJsonString());
break;
}
} catch (e, stackTrace) {
@ -361,7 +291,7 @@ class FaceMlService {
}
Future<void> clusterAllImages({
double minFaceScore = kMinHighQualityFaceScore,
double minFaceScore = kMinimumQualityFaceScore,
bool clusterInBuckets = true,
}) async {
_logger.info("`clusterAllImages()` called");
@ -370,6 +300,10 @@ class FaceMlService {
// Get a sense of the total number of faces in the database
final int totalFaces = await FaceMLDataDB.instance
.getTotalFaceCount(minFaceScore: minFaceScore);
// Get the current cluster statistics
final Map<int, (Uint8List, int)> oldClusterSummaries =
await FaceMLDataDB.instance.getAllClusterSummary();
if (clusterInBuckets) {
// read the creation times from Files DB, in a map from fileID to creation time
final fileIDToCreationTime =
@ -382,14 +316,14 @@ class FaceMlService {
int bucket = 1;
while (true) {
final faceIdToEmbeddingBucket =
await FaceMLDataDB.instance.getFaceEmbeddingMap(
final faceInfoForClustering =
await FaceMLDataDB.instance.getFaceInfoForClustering(
minScore: minFaceScore,
maxFaces: bucketSize,
offset: offset,
batchSize: batchSize,
);
if (faceIdToEmbeddingBucket.isEmpty) {
if (faceInfoForClustering.isEmpty) {
_logger.warning(
'faceIdToEmbeddingBucket is empty, this should ideally not happen as it should have stopped earlier. offset: $offset, totalFaces: $totalFaces',
);
@ -402,20 +336,24 @@ class FaceMlService {
break;
}
final faceIdToCluster =
final clusteringResult =
await FaceClusteringService.instance.predictLinear(
faceIdToEmbeddingBucket,
faceInfoForClustering,
fileIDToCreationTime: fileIDToCreationTime,
offset: offset,
oldClusterSummaries: oldClusterSummaries,
);
if (faceIdToCluster == null) {
if (clusteringResult == null) {
_logger.warning("faceIdToCluster is null");
return;
}
await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster);
await FaceMLDataDB.instance
.updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries!);
_logger.info(
'Done with clustering ${offset + faceIdToEmbeddingBucket.length} embeddings (${(100 * (offset + faceIdToEmbeddingBucket.length) / totalFaces).toStringAsFixed(0)}%) in bucket $bucket, offset: $offset',
'Done with clustering ${offset + faceInfoForClustering.length} embeddings (${(100 * (offset + faceInfoForClustering.length) / totalFaces).toStringAsFixed(0)}%) in bucket $bucket, offset: $offset',
);
if (offset + bucketSize >= totalFaces) {
_logger.info('All faces clustered');
@ -427,14 +365,14 @@ class FaceMlService {
} else {
// Read all the embeddings from the database, in a map from faceID to embedding
final clusterStartTime = DateTime.now();
final faceIdToEmbedding =
await FaceMLDataDB.instance.getFaceEmbeddingMap(
final faceInfoForClustering =
await FaceMLDataDB.instance.getFaceInfoForClustering(
minScore: minFaceScore,
maxFaces: totalFaces,
);
final gotFaceEmbeddingsTime = DateTime.now();
_logger.info(
'read embeddings ${faceIdToEmbedding.length} in ${gotFaceEmbeddingsTime.difference(clusterStartTime).inMilliseconds} ms',
'read embeddings ${faceInfoForClustering.length} in ${gotFaceEmbeddingsTime.difference(clusterStartTime).inMilliseconds} ms',
);
// Read the creation times from Files DB, in a map from fileID to creation time
@ -444,25 +382,29 @@ class FaceMlService {
'${DateTime.now().difference(gotFaceEmbeddingsTime).inMilliseconds} ms');
// Cluster the embeddings using the linear clustering algorithm, returning a map from faceID to clusterID
final faceIdToCluster =
final clusteringResult =
await FaceClusteringService.instance.predictLinear(
faceIdToEmbedding,
faceInfoForClustering,
fileIDToCreationTime: fileIDToCreationTime,
oldClusterSummaries: oldClusterSummaries,
);
if (faceIdToCluster == null) {
if (clusteringResult == null) {
_logger.warning("faceIdToCluster is null");
return;
}
final clusterDoneTime = DateTime.now();
_logger.info(
'done with clustering ${faceIdToEmbedding.length} in ${clusterDoneTime.difference(clusterStartTime).inSeconds} seconds ',
'done with clustering ${faceInfoForClustering.length} in ${clusterDoneTime.difference(clusterStartTime).inSeconds} seconds ',
);
// Store the updated clusterIDs in the database
_logger.info(
'Updating ${faceIdToCluster.length} FaceIDs with clusterIDs in the DB',
'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB',
);
await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster);
await FaceMLDataDB.instance
.updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries!);
_logger.info('Done updating FaceIDs with clusterIDs in the DB, in '
'${DateTime.now().difference(clusterDoneTime).inSeconds} seconds');
}
@ -875,6 +817,7 @@ class FaceMlService {
}
}
/// Analyzes the given image data by running the full pipeline for faces, using [analyzeImageSync] in the isolate.
Future<FaceMlResult?> analyzeImageInSingleIsolate(EnteFile enteFile) async {
_checkEnteFileForID(enteFile);
await ensureInitialized();
@ -931,6 +874,87 @@ class FaceMlService {
return result;
}
static Future<FaceMlResult> analyzeImageSync(Map args) async {
try {
final int enteFileID = args["enteFileID"] as int;
final String imagePath = args["filePath"] as String;
final int faceDetectionAddress = args["faceDetectionAddress"] as int;
final int faceEmbeddingAddress = args["faceEmbeddingAddress"] as int;
final resultBuilder = FaceMlResultBuilder.fromEnteFileID(enteFileID);
dev.log(
"Start analyzing image with uploadedFileID: $enteFileID inside the isolate",
);
final stopwatchTotal = Stopwatch()..start();
final stopwatch = Stopwatch()..start();
// Decode the image once to use for both face detection and alignment
final imageData = await File(imagePath).readAsBytes();
final image = await decodeImageFromData(imageData);
final ByteData imgByteData = await getByteDataFromImage(image);
dev.log('Reading and decoding image took '
'${stopwatch.elapsedMilliseconds} ms');
stopwatch.reset();
// Get the faces
final List<FaceDetectionRelative> faceDetectionResult =
await FaceMlService.detectFacesSync(
image,
imgByteData,
faceDetectionAddress,
resultBuilder: resultBuilder,
);
dev.log(
"${faceDetectionResult.length} faces detected with scores ${faceDetectionResult.map((e) => e.score).toList()}: completed `detectFacesSync` function, in "
"${stopwatch.elapsedMilliseconds} ms");
// If no faces were detected, return a result with no faces. Otherwise, continue.
if (faceDetectionResult.isEmpty) {
dev.log(
"No faceDetectionResult, Completed analyzing image with uploadedFileID $enteFileID, in "
"${stopwatch.elapsedMilliseconds} ms");
return resultBuilder.buildNoFaceDetected();
}
stopwatch.reset();
// Align the faces
final Float32List faceAlignmentResult =
await FaceMlService.alignFacesSync(
image,
imgByteData,
faceDetectionResult,
resultBuilder: resultBuilder,
);
dev.log("Completed `alignFacesSync` function, in "
"${stopwatch.elapsedMilliseconds} ms");
stopwatch.reset();
// Get the embeddings of the faces
final embeddings = await FaceMlService.embedFacesSync(
faceAlignmentResult,
faceEmbeddingAddress,
resultBuilder: resultBuilder,
);
dev.log("Completed `embedFacesSync` function, in "
"${stopwatch.elapsedMilliseconds} ms");
stopwatch.stop();
stopwatchTotal.stop();
dev.log("Finished Analyze image (${embeddings.length} faces) with "
"uploadedFileID $enteFileID, in "
"${stopwatchTotal.elapsedMilliseconds} ms");
return resultBuilder.build();
} catch (e, s) {
dev.log("Could not analyze image: \n e: $e \n s: $s");
rethrow;
}
}
Future<String?> _getImagePathForML(
EnteFile enteFile, {
FileDataForML typeOfData = FileDataForML.fileData,

View file

@ -5,6 +5,8 @@ import "package:flutter/foundation.dart";
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
// import "package:photos/events/files_updated_event.dart";
// import "package:photos/events/local_photos_updated_event.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart";
@ -115,17 +117,103 @@ class ClusterFeedbackService {
List<EnteFile> files,
PersonEntity p,
) async {
await FaceMLDataDB.instance.removeFilesFromPerson(files, p.remoteID);
Bus.instance.fire(PeopleChangedEvent());
try {
// Get the relevant faces to be removed
final faceIDs = await FaceMLDataDB.instance
.getFaceIDsForPerson(p.remoteID)
.then((iterable) => iterable.toList());
faceIDs.retainWhere((faceID) {
final fileID = getFileIdFromFaceId(faceID);
return files.any((file) => file.uploadedFileID == fileID);
});
final embeddings =
await FaceMLDataDB.instance.getFaceEmbeddingMapForFaces(faceIDs);
final fileIDToCreationTime =
await FilesDB.instance.getFileIDToCreationTime();
// Re-cluster within the deleted faces
final newFaceIdToClusterID =
await FaceClusteringService.instance.predictWithinClusterComputer(
embeddings,
fileIDToCreationTime: fileIDToCreationTime,
distanceThreshold: 0.20,
);
if (newFaceIdToClusterID == null || newFaceIdToClusterID.isEmpty) {
return;
}
// Update the deleted faces
await FaceMLDataDB.instance.forceUpdateClusterIds(newFaceIdToClusterID);
// Make sure the deleted faces don't get suggested in the future
final notClusterIdToPersonId = <int, String>{};
for (final clusterId in newFaceIdToClusterID.values.toSet()) {
notClusterIdToPersonId[clusterId] = p.remoteID;
}
await FaceMLDataDB.instance
.bulkCaptureNotPersonFeedback(notClusterIdToPersonId);
Bus.instance.fire(PeopleChangedEvent());
return;
} catch (e, s) {
_logger.severe("Error in removeFilesFromPerson", e, s);
rethrow;
}
}
Future<void> removeFilesFromCluster(
List<EnteFile> files,
int clusterID,
) async {
await FaceMLDataDB.instance.removeFilesFromCluster(files, clusterID);
Bus.instance.fire(PeopleChangedEvent());
return;
try {
// Get the relevant faces to be removed
final faceIDs = await FaceMLDataDB.instance
.getFaceIDsForCluster(clusterID)
.then((iterable) => iterable.toList());
faceIDs.retainWhere((faceID) {
final fileID = getFileIdFromFaceId(faceID);
return files.any((file) => file.uploadedFileID == fileID);
});
final embeddings =
await FaceMLDataDB.instance.getFaceEmbeddingMapForFaces(faceIDs);
final fileIDToCreationTime =
await FilesDB.instance.getFileIDToCreationTime();
// Re-cluster within the deleted faces
final newFaceIdToClusterID =
await FaceClusteringService.instance.predictWithinClusterComputer(
embeddings,
fileIDToCreationTime: fileIDToCreationTime,
distanceThreshold: 0.20,
);
if (newFaceIdToClusterID == null || newFaceIdToClusterID.isEmpty) {
return;
}
// Update the deleted faces
await FaceMLDataDB.instance.forceUpdateClusterIds(newFaceIdToClusterID);
Bus.instance.fire(
PeopleChangedEvent(
relevantFiles: files,
type: PeopleEventType.removedFilesFromCluster,
source: "$clusterID",
),
);
// Bus.instance.fire(
// LocalPhotosUpdatedEvent(
// files,
// type: EventType.peopleClusterChanged,
// source: "$clusterID",
// ),
// );
return;
} catch (e, s) {
_logger.severe("Error in removeFilesFromCluster", e, s);
rethrow;
}
}
Future<void> addFilesToCluster(List<String> faceIDs, int clusterID) async {
@ -194,7 +282,7 @@ class ClusterFeedbackService {
// TODO: iterate over this method to find sweet spot
Future<Map<int, List<String>>> breakUpCluster(
int clusterID, {
useDbscan = false,
bool useDbscan = false,
}) async {
_logger.info(
'breakUpCluster called for cluster $clusterID with dbscan $useDbscan',
@ -203,10 +291,8 @@ class ClusterFeedbackService {
final faceIDs = await faceMlDb.getFaceIDsForCluster(clusterID);
final originalFaceIDsSet = faceIDs.toSet();
final fileIDs = faceIDs.map((e) => getFileIdFromFaceId(e)).toList();
final embeddings = await faceMlDb.getFaceEmbeddingMapForFile(fileIDs);
embeddings.removeWhere((key, value) => !faceIDs.contains(key));
final embeddings = await faceMlDb.getFaceEmbeddingMapForFaces(faceIDs);
final fileIDToCreationTime =
await FilesDB.instance.getFileIDToCreationTime();
@ -232,18 +318,14 @@ class ClusterFeedbackService {
maxClusterID++;
}
} else {
final clusteringInput = embeddings.map((key, value) {
return MapEntry(key, (null, value));
});
final faceIdToCluster =
await FaceClusteringService.instance.predictLinear(
clusteringInput,
await FaceClusteringService.instance.predictWithinClusterComputer(
embeddings,
fileIDToCreationTime: fileIDToCreationTime,
distanceThreshold: 0.23,
distanceThreshold: 0.22,
);
if (faceIdToCluster == null) {
if (faceIdToCluster == null || faceIdToCluster.isEmpty) {
_logger.info('No clusters found');
return {};
} else {
@ -295,6 +377,62 @@ class ClusterFeedbackService {
return clusterIdToFaceIds;
}
/// WARNING: this method is purely for debugging purposes, never use in production
Future<void> createFakeClustersByBlurValue() async {
try {
// Delete old clusters
await FaceMLDataDB.instance.resetClusterIDs();
await FaceMLDataDB.instance.dropClustersAndPersonTable();
final List<PersonEntity> persons =
await PersonService.instance.getPersons();
for (final PersonEntity p in persons) {
await PersonService.instance.deletePerson(p.remoteID);
}
// Create new fake clusters based on blur value. One for values between 0 and 10, one for 10-20, etc till 200
final int startClusterID = DateTime.now().microsecondsSinceEpoch;
final faceIDsToBlurValues =
await FaceMLDataDB.instance.getFaceIDsToBlurValues(200);
final faceIdToCluster = <String, int>{};
for (final entry in faceIDsToBlurValues.entries) {
final faceID = entry.key;
final blurValue = entry.value;
final newClusterID = startClusterID + blurValue ~/ 10;
faceIdToCluster[faceID] = newClusterID;
}
await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster);
Bus.instance.fire(PeopleChangedEvent());
} catch (e, s) {
_logger.severe("Error in createFakeClustersByBlurValue", e, s);
rethrow;
}
}
Future<void> debugLogClusterBlurValues(
int clusterID, {
int? clusterSize,
}) async {
final List<double> blurValues = await FaceMLDataDB.instance
.getBlurValuesForCluster(clusterID)
.then((value) => value.toList());
// Round the blur values to integers
final blurValuesIntegers =
blurValues.map((value) => value.round()).toList();
// Sort the blur values in ascending order
blurValuesIntegers.sort();
// Log the sorted blur values
_logger.info(
"Blur values for cluster $clusterID${clusterSize != null ? ' with $clusterSize photos' : ''}: $blurValuesIntegers",
);
return;
}
/// Returns a map of person's clusterID to map of closest clusterID to with disstance
Future<Map<int, List<(int, double)>>> getSuggestionsUsingMean(
PersonEntity p, {
@ -523,7 +661,7 @@ class ClusterFeedbackService {
);
final Map<int, (Uint8List, int)> clusterToSummary =
await faceMlDb.clusterSummaryAll();
await faceMlDb.getAllClusterSummary();
final Map<int, (Uint8List, int)> updatesForClusterSummary = {};
final Map<int, List<double>> clusterAvg = {};
@ -714,7 +852,7 @@ class ClusterFeedbackService {
// Get the cluster averages for the person's clusters and the suggestions' clusters
final Map<int, (Uint8List, int)> clusterToSummary =
await faceMlDb.clusterSummaryAll();
await faceMlDb.getAllClusterSummary();
// Calculate the avg embedding of the person
final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID);

View file

@ -824,7 +824,7 @@ class SearchService {
"Cluster $clusterId should not have person id ${clusterIDToPersonID[clusterId]}",
);
}
if (files.length < 3 && sortedClusterIds.length > 3) {
if (files.length < 20 && sortedClusterIds.length > 3) {
continue;
}
facesResult.add(

View file

@ -8,6 +8,7 @@ import "package:photos/events/people_changed_event.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart';
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/theme/ente_theme.dart';
import 'package:photos/ui/components/captioned_text_widget.dart';
@ -284,6 +285,34 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Rank blurs",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
await showChoiceDialog(
context,
title: "Are you sure?",
body:
"This will delete all clusters and put blurry faces in separate clusters per ten points.",
firstButtonLabel: "Yes, confirm",
firstButtonOnTap: () async {
try {
await ClusterFeedbackService.instance
.createFakeClustersByBlurValue();
showShortToast(context, "Done");
} catch (e, s) {
_logger.warning('Failed to rank faces on blur values ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
);
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Drop embeddings & feedback",

View file

@ -9,6 +9,7 @@ import "package:photos/face/db.dart";
import "package:photos/face/model/face.dart";
import "package:photos/face/model/person.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart";
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/search_service.dart";
import "package:photos/theme/ente_theme.dart";
@ -47,7 +48,7 @@ class _FaceWidgetState extends State<FaceWidget> {
@override
Widget build(BuildContext context) {
if (Platform.isIOS || Platform.isAndroid) {
if (Platform.isIOS) {
return FutureBuilder<Uint8List?>(
future: getFaceCrop(),
builder: (context, snapshot) {
@ -164,19 +165,19 @@ class _FaceWidgetState extends State<FaceWidget> {
),
if (kDebugMode)
Text(
'B: ${widget.face.blur.toStringAsFixed(3)}',
'B: ${widget.face.blur.toStringAsFixed(0)}',
style: Theme.of(context).textTheme.bodySmall,
maxLines: 1,
),
if (kDebugMode)
Text(
'V: ${widget.face.visibility}',
'D: ${widget.face.detection.getFaceDirection().toDirectionString()}',
style: Theme.of(context).textTheme.bodySmall,
maxLines: 1,
),
if (kDebugMode)
Text(
'A: ${widget.face.area()}',
'Sideways: ${widget.face.detection.faceIsSideways().toString()}',
style: Theme.of(context).textTheme.bodySmall,
maxLines: 1,
),
@ -303,6 +304,24 @@ class _FaceWidgetState extends State<FaceWidget> {
style: Theme.of(context).textTheme.bodySmall,
maxLines: 1,
),
if (kDebugMode)
Text(
'B: ${widget.face.blur.toStringAsFixed(0)}',
style: Theme.of(context).textTheme.bodySmall,
maxLines: 1,
),
if (kDebugMode)
Text(
'D: ${widget.face.detection.getFaceDirection().toDirectionString()}',
style: Theme.of(context).textTheme.bodySmall,
maxLines: 1,
),
if (kDebugMode)
Text(
'Sideways: ${widget.face.detection.faceIsSideways().toString()}',
style: Theme.of(context).textTheme.bodySmall,
maxLines: 1,
),
],
),
);

View file

@ -1,10 +1,12 @@
import "dart:async";
import "package:flutter/foundation.dart";
import 'package:flutter/material.dart';
import "package:flutter_animate/flutter_animate.dart";
import 'package:photos/core/event_bus.dart';
import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/local_photos_updated_event.dart';
import "package:photos/events/people_changed_event.dart";
import "package:photos/face/model/person.dart";
import "package:photos/generated/l10n.dart";
import 'package:photos/models/file/file.dart';
@ -51,6 +53,7 @@ class _ClusterPageState extends State<ClusterPage> {
final _selectedFiles = SelectedFiles();
late final List<EnteFile> files;
late final StreamSubscription<LocalPhotosUpdatedEvent> _filesUpdatedEvent;
late final StreamSubscription<PeopleChangedEvent> _peopleChangedEvent;
@override
void initState() {
@ -69,11 +72,27 @@ class _ClusterPageState extends State<ClusterPage> {
setState(() {});
}
});
_peopleChangedEvent = Bus.instance.on<PeopleChangedEvent>().listen((event) {
if (event.type == PeopleEventType.removedFilesFromCluster &&
(event.source == widget.clusterID.toString())) {
for (var updatedFile in event.relevantFiles!) {
files.remove(updatedFile);
}
setState(() {});
}
});
kDebugMode
? ClusterFeedbackService.instance.debugLogClusterBlurValues(
widget.clusterID,
clusterSize: files.length,
)
: null;
}
@override
void dispose() {
_filesUpdatedEvent.cancel();
_peopleChangedEvent.cancel();
super.dispose();
}
@ -96,10 +115,12 @@ class _ClusterPageState extends State<ClusterPage> {
);
},
reloadEvent: Bus.instance.on<LocalPhotosUpdatedEvent>(),
forceReloadEvents: [Bus.instance.on<PeopleChangedEvent>()],
removalEventTypes: const {
EventType.deletedFromRemote,
EventType.deletedFromEverywhere,
EventType.hide,
EventType.peopleClusterChanged,
},
tagPrefix: widget.tagPrefix + widget.tagPrefix,
selectedFiles: _selectedFiles,
@ -111,9 +132,10 @@ class _ClusterPageState extends State<ClusterPage> {
preferredSize: const Size.fromHeight(50.0),
child: ClusterAppBar(
SearchResultPage.appBarType,
"${widget.searchResult.length} memories${widget.appendTitle}",
"${files.length} memories${widget.appendTitle}",
_selectedFiles,
widget.clusterID,
key: ValueKey(files.length),
),
),
body: Column(

View file

@ -1099,19 +1099,16 @@ Future<(Float32List, List<AlignmentResult>, List<bool>, List<double>, Size)>
imageHeight: image.height,
);
final List<List<List<double>>> faceLandmarks =
absoluteFaces.map((face) => face.allKeypoints).toList();
final alignedImagesFloat32List =
Float32List(3 * width * height * faceLandmarks.length);
Float32List(3 * width * height * absoluteFaces.length);
final alignmentResults = <AlignmentResult>[];
final isBlurs = <bool>[];
final blurValues = <double>[];
int alignedImageIndex = 0;
for (final faceLandmark in faceLandmarks) {
for (final face in absoluteFaces) {
final (alignmentResult, correctlyEstimated) =
SimilarityTransform.instance.estimate(faceLandmark);
SimilarityTransform.instance.estimate(face.allKeypoints);
if (!correctlyEstimated) {
alignedImageIndex += 3 * width * height;
alignmentResults.add(AlignmentResult.empty());
@ -1137,7 +1134,7 @@ Future<(Float32List, List<AlignmentResult>, List<bool>, List<double>, Size)>
final grayscalems = blurDetectionStopwatch.elapsedMilliseconds;
log('creating grayscale matrix took $grayscalems ms');
final (isBlur, blurValue) = await BlurDetectionService.instance
.predictIsBlurGrayLaplacian(faceGrayMatrix);
.predictIsBlurGrayLaplacian(faceGrayMatrix, faceDirection: face.getFaceDirection());
final blurms = blurDetectionStopwatch.elapsedMilliseconds - grayscalems;
log('blur detection took $blurms ms');
log(