beenull/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'mobile_face' of https://github.com/ente-io/auth into mobile_face

Browse source
This commit is contained in:
Neeraj Gupta 2024-04-29 17:18:32 +05:30
commit 7d2633190f
21 changed files with 1248 additions and 592 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

626
mobile/lib/face/db.dart
View file

@ -1,5 +1,4 @@
import 'dart:async';
import "dart:io" show Directory;
import "dart:math";
import "package:collection/collection.dart";
@ -14,14 +13,16 @@ import "package:photos/face/model/face.dart";
import "package:photos/models/file/file.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
import 'package:sqflite/sqflite.dart';
import 'package:sqlite_async/sqlite_async.dart' as sqlite_async;
import 'package:sqlite_async/sqlite_async.dart';
/// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`.
/// Stores all data for the FacesML-related features. The database can be accessed by `FaceMLDataDB.instance.database`.
///
/// This includes:
/// [facesTable] - Stores all the detected faces and its embeddings in the images.
/// [personTable] - Stores all the clusters of faces which are considered to be the same person.
/// [createFaceClustersTable] - Stores all the mappings from the faces (faceID) to the clusters (clusterID).
/// [clusterPersonTable] - Stores all the clusters that are mapped to a certain person.
/// [clusterSummaryTable] - Stores a summary of each cluster, containg the mean embedding and the number of faces in the cluster.
/// [notPersonFeedback] - Stores the clusters that are confirmed not to belong to a certain person by the user
class FaceMLDataDB {
static final Logger _logger = Logger("FaceMLDataDB");
@ -33,75 +34,81 @@ class FaceMLDataDB {
static final FaceMLDataDB instance = FaceMLDataDB._privateConstructor();
// only have a single app-wide reference to the database
static Future<Database>? _dbFuture;
static Future<sqlite_async.SqliteDatabase>? _sqliteAsyncDBFuture;
static Future<SqliteDatabase>? _sqliteAsyncDBFuture;
Future<Database> get database async {
_dbFuture ??= _initDatabase();
return _dbFuture!;
}
Future<sqlite_async.SqliteDatabase> get sqliteAsyncDB async {
Future<SqliteDatabase> get asyncDB async {
_sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase();
return _sqliteAsyncDBFuture!;
}
Future<Database> _initDatabase() async {
Future<SqliteDatabase> _initSqliteAsyncDatabase() async {
final documentsDirectory = await getApplicationDocumentsDirectory();
final String databaseDirectory =
join(documentsDirectory.path, _databaseName);
return await openDatabase(
databaseDirectory,
version: _databaseVersion,
onCreate: _onCreate,
);
}
Future<sqlite_async.SqliteDatabase> _initSqliteAsyncDatabase() async {
final Directory documentsDirectory =
await getApplicationDocumentsDirectory();
final String databaseDirectory =
join(documentsDirectory.path, _databaseName);
_logger.info("Opening sqlite_async access: DB path " + databaseDirectory);
return sqlite_async.SqliteDatabase(path: databaseDirectory, maxReaders: 1);
final asyncDBConnection =
SqliteDatabase(path: databaseDirectory, maxReaders: 2);
await _onCreate(asyncDBConnection);
return asyncDBConnection;
}
Future _onCreate(Database db, int version) async {
await db.execute(createFacesTable);
await db.execute(createFaceClustersTable);
await db.execute(createClusterPersonTable);
await db.execute(createClusterSummaryTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(fcClusterIDIndex);
Future<void> _onCreate(SqliteDatabase asyncDBConnection) async {
final migrations = SqliteMigrations()
..add(
SqliteMigration(_databaseVersion, (tx) async {
await tx.execute(createFacesTable);
await tx.execute(createFaceClustersTable);
await tx.execute(createClusterPersonTable);
await tx.execute(createClusterSummaryTable);
await tx.execute(createNotPersonFeedbackTable);
await tx.execute(fcClusterIDIndex);
}),
);
await migrations.migrate(asyncDBConnection);
}
// bulkInsertFaces inserts the faces in the database in batches of 1000.
// This is done to avoid the error "too many SQL variables" when inserting
// a large number of faces.
Future<void> bulkInsertFaces(List<Face> faces) async {
final db = await instance.database;
final db = await instance.asyncDB;
const batchSize = 500;
final numBatches = (faces.length / batchSize).ceil();
for (int i = 0; i < numBatches; i++) {
final start = i * batchSize;
final end = min((i + 1) * batchSize, faces.length);
final batch = faces.sublist(start, end);
final batchInsert = db.batch();
for (final face in batch) {
batchInsert.insert(
facesTable,
mapRemoteToFaceDB(face),
conflictAlgorithm: ConflictAlgorithm.ignore,
);
}
await batchInsert.commit(noResult: true);
const String sql = '''
INSERT INTO $facesTable (
$fileIDColumn, $faceIDColumn, $faceDetectionColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways, $imageHeight, $imageWidth, $mlVersionColumn
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT($fileIDColumn, $faceIDColumn) DO UPDATE SET $faceIDColumn = excluded.$faceIDColumn, $faceDetectionColumn = excluded.$faceDetectionColumn, $faceEmbeddingBlob = excluded.$faceEmbeddingBlob, $faceScore = excluded.$faceScore, $faceBlur = excluded.$faceBlur, $isSideways = excluded.$isSideways, $imageHeight = excluded.$imageHeight, $imageWidth = excluded.$imageWidth, $mlVersionColumn = excluded.$mlVersionColumn
''';
final parameterSets = batch.map((face) {
final map = mapRemoteToFaceDB(face);
return [
map[fileIDColumn],
map[faceIDColumn],
map[faceDetectionColumn],
map[faceEmbeddingBlob],
map[faceScore],
map[faceBlur],
map[isSideways],
map[imageHeight],
map[imageWidth],
map[mlVersionColumn],
];
}).toList();
await db.executeBatch(sql, parameterSets);
}
}
Future<void> updateClusterIdToFaceId(
Future<void> updateFaceIdToClusterId(
Map<String, int> faceIDToClusterID,
) async {
final db = await instance.database;
final db = await instance.asyncDB;
const batchSize = 500;
final numBatches = (faceIDToClusterID.length / batchSize).ceil();
for (int i = 0; i < numBatches; i++) {
@ -109,24 +116,20 @@ class FaceMLDataDB {
final end = min((i + 1) * batchSize, faceIDToClusterID.length);
final batch = faceIDToClusterID.entries.toList().sublist(start, end);
final batchUpdate = db.batch();
const String sql = '''
INSERT INTO $faceClustersTable ($fcFaceId, $fcClusterID)
VALUES (?, ?)
ON CONFLICT($fcFaceId) DO UPDATE SET $fcClusterID = excluded.$fcClusterID
''';
final parameterSets = batch.map((e) => [e.key, e.value]).toList();
for (final entry in batch) {
final faceID = entry.key;
final clusterID = entry.value;
batchUpdate.insert(
faceClustersTable,
{fcClusterID: clusterID, fcFaceId: faceID},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
await batchUpdate.commit(noResult: true);
await db.executeBatch(sql, parameterSets);
}
}
/// Returns a map of fileID to the indexed ML version
Future<Map<int, int>> getIndexedFileIds() async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fileIDColumn, $mlVersionColumn FROM $facesTable',
);
@ -138,7 +141,7 @@ class FaceMLDataDB {
}
Future<int> getIndexedFileCount() async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT COUNT(DISTINCT $fileIDColumn) as count FROM $facesTable',
);
@ -146,8 +149,8 @@ class FaceMLDataDB {
}
Future<Map<int, int>> clusterIdToFaceCount() async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, COUNT(*) as count FROM $faceClustersTable where $fcClusterID IS NOT NULL GROUP BY $fcClusterID ',
);
final Map<int, int> result = {};
@ -158,15 +161,15 @@ class FaceMLDataDB {
}
Future<Set<int>> getPersonIgnoredClusters(String personID) async {
final db = await instance.database;
final db = await instance.asyncDB;
// find out clusterIds that are assigned to other persons using the clusters table
final List<Map<String, dynamic>> maps = await db.rawQuery(
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL',
[personID],
);
final Set<int> ignoredClusterIDs =
maps.map((e) => e[clusterIDColumn] as int).toSet();
final List<Map<String, dynamic>> rejectMaps = await db.rawQuery(
final List<Map<String, dynamic>> rejectMaps = await db.getAll(
'SELECT $clusterIDColumn FROM $notPersonFeedback WHERE $personIdColumn = ?',
[personID],
);
@ -176,8 +179,8 @@ class FaceMLDataDB {
}
Future<Set<int>> getPersonClusterIDs(String personID) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?',
[personID],
);
@ -185,20 +188,21 @@ class FaceMLDataDB {
}
Future<void> clearTable() async {
final db = await instance.database;
await db.delete(facesTable);
await db.delete(clusterPersonTable);
await db.delete(clusterSummaryTable);
await db.delete(personTable);
await db.delete(notPersonFeedback);
final db = await instance.asyncDB;
await db.execute(deleteFacesTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropClusterSummaryTable);
await db.execute(deletePersonTable);
await db.execute(dropNotPersonFeedbackTable);
}
Future<Iterable<Uint8List>> getFaceEmbeddingsForCluster(
int clusterID, {
int? limit,
}) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceEmbeddingBlob FROM $facesTable WHERE $faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID = ?) ${limit != null ? 'LIMIT $limit' : ''}',
[clusterID],
);
@ -209,7 +213,7 @@ class FaceMLDataDB {
Iterable<int> clusterIDs, {
int? limit,
}) async {
final db = await instance.database;
final db = await instance.asyncDB;
final Map<int, List<Uint8List>> result = {};
final selectQuery = '''
@ -220,7 +224,7 @@ class FaceMLDataDB {
${limit != null ? 'LIMIT $limit' : ''}
''';
final List<Map<String, dynamic>> maps = await db.rawQuery(selectQuery);
final List<Map<String, dynamic>> maps = await db.getAll(selectQuery);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
@ -238,7 +242,7 @@ class FaceMLDataDB {
int? clusterID,
}) async {
// read person from db
final db = await instance.database;
final db = await instance.asyncDB;
if (personID != null) {
final List<int> fileId = [recentFileID];
int? avatarFileId;
@ -248,15 +252,18 @@ class FaceMLDataDB {
fileId.add(avatarFileId);
}
}
final cluterRows = await db.query(
clusterPersonTable,
columns: [clusterIDColumn],
where: '$personIdColumn = ?',
whereArgs: [personID],
const String queryClusterID = '''
SELECT $clusterIDColumn
FROM $clusterPersonTable
WHERE $personIdColumn = ?
''';
final clusterRows = await db.getAll(
queryClusterID,
[personID],
);
final clusterIDs =
cluterRows.map((e) => e[clusterIDColumn] as int).toList();
final List<Map<String, dynamic>> faceMaps = await db.rawQuery(
clusterRows.map((e) => e[clusterIDColumn] as int).toList();
final List<Map<String, dynamic>> faceMaps = await db.getAll(
'SELECT * FROM $facesTable where '
'$faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID IN (${clusterIDs.join(",")}))'
'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinimumQualityFaceScore ORDER BY $faceScore DESC',
@ -274,11 +281,14 @@ class FaceMLDataDB {
}
}
if (clusterID != null) {
final List<Map<String, dynamic>> faceMaps = await db.query(
faceClustersTable,
columns: [fcFaceId],
where: '$fcClusterID = ?',
whereArgs: [clusterID],
const String queryFaceID = '''
SELECT $fcFaceId
FROM $faceClustersTable
WHERE $fcClusterID = ?
''';
final List<Map<String, dynamic>> faceMaps = await db.getAll(
queryFaceID,
[clusterID],
);
final List<Face>? faces = await getFacesForGivenFileID(recentFileID);
if (faces != null) {
@ -297,22 +307,14 @@ class FaceMLDataDB {
}
Future<List<Face>?> getFacesForGivenFileID(int fileUploadID) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.query(
facesTable,
columns: [
faceIDColumn,
fileIDColumn,
faceEmbeddingBlob,
faceScore,
faceDetectionColumn,
faceBlur,
imageHeight,
imageWidth,
mlVersionColumn,
],
where: '$fileIDColumn = ?',
whereArgs: [fileUploadID],
final db = await instance.asyncDB;
const String query = '''
SELECT * FROM $facesTable
WHERE $fileIDColumn = ?
''';
final List<Map<String, dynamic>> maps = await db.getAll(
query,
[fileUploadID],
);
if (maps.isEmpty) {
return null;
@ -321,8 +323,8 @@ class FaceMLDataDB {
}
Future<Face?> getFaceForFaceID(String faceID) async {
final db = await instance.database;
final result = await db.rawQuery(
final db = await instance.asyncDB;
final result = await db.getAll(
'SELECT * FROM $facesTable where $faceIDColumn = ?',
[faceID],
);
@ -332,8 +334,50 @@ class FaceMLDataDB {
return mapRowToFace(result.first);
}
Future<Map<int, Iterable<String>>> getClusterToFaceIDs(
Set<int> clusterIDs,
) async {
final db = await instance.asyncDB;
final Map<int, List<String>> result = {};
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable WHERE $fcClusterID IN (${clusterIDs.join(",")})',
);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final faceID = map[fcFaceId] as String;
result.putIfAbsent(clusterID, () => <String>[]).add(faceID);
}
return result;
}
Future<int?> getClusterIDForFaceID(String faceID) async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID FROM $faceClustersTable WHERE $fcFaceId = ?',
[faceID],
);
if (maps.isEmpty) {
return null;
}
return maps.first[fcClusterID] as int;
}
Future<Map<int, Iterable<String>>> getAllClusterIdToFaceIDs() async {
final db = await instance.asyncDB;
final Map<int, List<String>> result = {};
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable',
);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final faceID = map[fcFaceId] as String;
result.putIfAbsent(clusterID, () => <String>[]).add(faceID);
}
return result;
}
Future<Iterable<String>> getFaceIDsForCluster(int clusterID) async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable '
'WHERE $faceClustersTable.$fcClusterID = ?',
@ -343,7 +387,7 @@ class FaceMLDataDB {
}
Future<Iterable<String>> getFaceIDsForPerson(String personID) async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final faceIdsResult = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
@ -354,7 +398,7 @@ class FaceMLDataDB {
}
Future<Iterable<double>> getBlurValuesForCluster(int clusterID) async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
const String query = '''
SELECT $facesTable.$faceBlur
FROM $facesTable
@ -376,7 +420,7 @@ class FaceMLDataDB {
Future<Map<String, double>> getFaceIDsToBlurValues(
int maxBlurValue,
) async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceIDColumn, $faceBlur FROM $facesTable WHERE $faceBlur < $maxBlurValue AND $faceBlur > 1 ORDER BY $faceBlur ASC',
);
@ -390,8 +434,8 @@ class FaceMLDataDB {
Future<Map<String, int?>> getFaceIdsToClusterIds(
Iterable<String> faceIds,
) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcFaceId, $fcClusterID FROM $faceClustersTable where $fcFaceId IN (${faceIds.map((id) => "'$id'").join(",")})',
);
final Map<String, int?> result = {};
@ -403,8 +447,8 @@ class FaceMLDataDB {
Future<Map<int, Set<int>>> getFileIdToClusterIds() async {
final Map<int, Set<int>> result = {};
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable',
);
@ -421,36 +465,31 @@ class FaceMLDataDB {
Future<void> forceUpdateClusterIds(
Map<String, int> faceIDToClusterID,
) async {
final db = await instance.database;
final db = await instance.asyncDB;
// Start a batch
final batch = db.batch();
for (final map in faceIDToClusterID.entries) {
final faceID = map.key;
final clusterID = map.value;
batch.insert(
faceClustersTable,
{fcFaceId: faceID, fcClusterID: clusterID},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
// Commit the batch
await batch.commit(noResult: true);
const String sql = '''
INSERT INTO $faceClustersTable ($fcFaceId, $fcClusterID)
VALUES (?, ?)
ON CONFLICT($fcFaceId) DO UPDATE SET $fcClusterID = excluded.$fcClusterID
''';
final parameterSets =
faceIDToClusterID.entries.map((e) => [e.key, e.value]).toList();
await db.executeBatch(sql, parameterSets);
}
Future<void> removePerson(String personID) async {
final db = await instance.database;
await db.delete(
clusterPersonTable,
where: '$personIdColumn = ?',
whereArgs: [personID],
);
await db.delete(
notPersonFeedback,
where: '$personIdColumn = ?',
whereArgs: [personID],
);
final db = await instance.asyncDB;
await db.writeTransaction((tx) async {
await tx.execute(
'DELETE FROM $clusterPersonTable WHERE $personIdColumn = ?',
[personID],
);
await tx.execute(
'DELETE FROM $notPersonFeedback WHERE $personIdColumn = ?',
[personID],
);
});
}
Future<Set<FaceInfoForClustering>> getFaceInfoForClustering({
@ -464,7 +503,7 @@ class FaceMLDataDB {
w.logAndReset(
'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
);
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final Set<FaceInfoForClustering> result = {};
while (true) {
@ -519,7 +558,7 @@ class FaceMLDataDB {
w.logAndReset(
'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
);
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final Map<String, (int?, Uint8List)> result = {};
while (true) {
@ -563,7 +602,7 @@ class FaceMLDataDB {
List<int> fileIDs,
) async {
_logger.info('reading face embeddings for ${fileIDs.length} files');
final db = await instance.database;
final db = await instance.asyncDB;
// Define the batch size
const batchSize = 10000;
@ -572,15 +611,23 @@ class FaceMLDataDB {
final Map<String, Uint8List> result = {};
while (true) {
// Query a batch of rows
final List<Map<String, dynamic>> maps = await db.query(
facesTable,
columns: [faceIDColumn, faceEmbeddingBlob],
where:
'$faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})',
limit: batchSize,
offset: offset,
orderBy: '$faceIDColumn DESC',
);
final List<Map<String, dynamic>> maps = await db.getAll('''
SELECT $faceIDColumn, $faceEmbeddingBlob
FROM $facesTable
WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})
ORDER BY $faceIDColumn DESC
LIMIT $batchSize OFFSET $offset
''');
// final List<Map<String, dynamic>> maps = await db.query(
// facesTable,
// columns: [faceIDColumn, faceEmbeddingBlob],
// where:
// '$faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})',
// limit: batchSize,
// offset: offset,
// orderBy: '$faceIDColumn DESC',
// );
// Break the loop if no more rows
if (maps.isEmpty) {
break;
@ -602,7 +649,7 @@ class FaceMLDataDB {
Iterable<String> faceIDs,
) async {
_logger.info('reading face embeddings for ${faceIDs.length} faces');
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
// Define the batch size
const batchSize = 10000;
@ -639,7 +686,7 @@ class FaceMLDataDB {
Future<int> getTotalFaceCount({
double minFaceScore = kMinimumQualityFaceScore,
}) async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianHardThreshold',
);
@ -647,7 +694,7 @@ class FaceMLDataDB {
}
Future<double> getClusteredToTotalFacesRatio() async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final List<Map<String, dynamic>> totalFacesMaps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold',
@ -665,105 +712,107 @@ class FaceMLDataDB {
Future<int> getBlurryFaceCount([
int blurThreshold = kLaplacianHardThreshold,
]) async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceBlur <= $blurThreshold AND $faceScore > $kMinimumQualityFaceScore',
);
return maps.first['count'] as int;
}
Future<void> resetClusterIDs() async {
final db = await instance.database;
await db.execute(dropFaceClustersTable);
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex);
try {
final db = await instance.asyncDB;
await db.execute(dropFaceClustersTable);
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex);
} catch (e, s) {
_logger.severe('Error resetting clusterIDs', e, s);
}
}
Future<void> assignClusterToPerson({
required String personID,
required int clusterID,
}) async {
final db = await instance.database;
await db.insert(
clusterPersonTable,
{
personIdColumn: personID,
clusterIDColumn: clusterID,
},
);
final db = await instance.asyncDB;
const String sql = '''
INSERT INTO $clusterPersonTable ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
''';
await db.execute(sql, [personID, clusterID]);
}
Future<void> bulkAssignClusterToPersonID(
Map<int, String> clusterToPersonID,
) async {
final db = await instance.database;
final batch = db.batch();
for (final entry in clusterToPersonID.entries) {
final clusterID = entry.key;
final personID = entry.value;
batch.insert(
clusterPersonTable,
{
personIdColumn: personID,
clusterIDColumn: clusterID,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
await batch.commit(noResult: true);
final db = await instance.asyncDB;
const String sql = '''
INSERT INTO $clusterPersonTable ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
''';
final parameterSets =
clusterToPersonID.entries.map((e) => [e.value, e.key]).toList();
await db.executeBatch(sql, parameterSets);
// final batch = db.batch();
// for (final entry in clusterToPersonID.entries) {
// final clusterID = entry.key;
// final personID = entry.value;
// batch.insert(
// clusterPersonTable,
// {
// personIdColumn: personID,
// clusterIDColumn: clusterID,
// },
// conflictAlgorithm: ConflictAlgorithm.replace,
// );
// }
// await batch.commit(noResult: true);
}
Future<void> captureNotPersonFeedback({
required String personID,
required int clusterID,
}) async {
final db = await instance.database;
await db.insert(
notPersonFeedback,
{
personIdColumn: personID,
clusterIDColumn: clusterID,
},
);
final db = await instance.asyncDB;
const String sql = '''
INSERT INTO $notPersonFeedback ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
''';
await db.execute(sql, [personID, clusterID]);
}
Future<void> bulkCaptureNotPersonFeedback(
Map<int, String> clusterToPersonID,
) async {
final db = await instance.database;
final batch = db.batch();
for (final entry in clusterToPersonID.entries) {
final clusterID = entry.key;
final personID = entry.value;
batch.insert(
notPersonFeedback,
{
personIdColumn: personID,
clusterIDColumn: clusterID,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
await batch.commit(noResult: true);
final db = await instance.asyncDB;
const String sql = '''
INSERT INTO $notPersonFeedback ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
''';
final parameterSets =
clusterToPersonID.entries.map((e) => [e.value, e.key]).toList();
await db.executeBatch(sql, parameterSets);
}
Future<int> removeClusterToPerson({
Future<void> removeClusterToPerson({
required String personID,
required int clusterID,
}) async {
final db = await instance.database;
return db.delete(
clusterPersonTable,
where: '$personIdColumn = ? AND $clusterIDColumn = ?',
whereArgs: [personID, clusterID],
);
final db = await instance.asyncDB;
const String sql = '''
DELETE FROM $clusterPersonTable WHERE $personIdColumn = ? AND $clusterIDColumn = ?
''';
await db.execute(sql, [personID, clusterID]);
}
// for a given personID, return a map of clusterID to fileIDs using join query
Future<Map<int, Set<int>>> getFileIdToClusterIDSet(String personID) {
final db = instance.database;
final db = instance.asyncDB;
return db.then((db) async {
final List<Map<String, dynamic>> maps = await db.rawQuery(
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceClustersTable.$fcClusterID, $fcFaceId FROM $faceClustersTable '
'INNER JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
@ -784,9 +833,9 @@ class FaceMLDataDB {
Future<Map<int, Set<int>>> getFileIdToClusterIDSetForCluster(
Set<int> clusterIDs,
) {
final db = instance.database;
final db = instance.asyncDB;
return db.then((db) async {
final List<Map<String, dynamic>> maps = await db.rawQuery(
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable '
'WHERE $fcClusterID IN (${clusterIDs.join(",")})',
);
@ -802,37 +851,57 @@ class FaceMLDataDB {
}
Future<void> clusterSummaryUpdate(Map<int, (Uint8List, int)> summary) async {
final db = await instance.database;
var batch = db.batch();
final db = await instance.asyncDB;
const String sql = '''
INSERT INTO $clusterSummaryTable ($clusterIDColumn, $avgColumn, $countColumn) VALUES (?, ?, ?) ON CONFLICT($clusterIDColumn) DO UPDATE SET $avgColumn = excluded.$avgColumn, $countColumn = excluded.$countColumn
''';
final List<List<Object?>> parameterSets = [];
int batchCounter = 0;
for (final entry in summary.entries) {
if (batchCounter == 400) {
await batch.commit(noResult: true);
batch = db.batch();
await db.executeBatch(sql, parameterSets);
batchCounter = 0;
parameterSets.clear();
}
final int cluserID = entry.key;
final int clusterID = entry.key;
final int count = entry.value.$2;
final Uint8List avg = entry.value.$1;
batch.insert(
clusterSummaryTable,
{
clusterIDColumn: cluserID,
avgColumn: avg,
countColumn: count,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
parameterSets.add([clusterID, avg, count]);
batchCounter++;
}
await batch.commit(noResult: true);
await db.executeBatch(sql, parameterSets);
// var batch = db.batch();
// int batchCounter = 0;
// for (final entry in summary.entries) {
// if (batchCounter == 400) {
// await batch.commit(noResult: true);
// batch = db.batch();
// batchCounter = 0;
// }
// final int cluserID = entry.key;
// final int count = entry.value.$2;
// final Uint8List avg = entry.value.$1;
// batch.insert(
// clusterSummaryTable,
// {
// clusterIDColumn: cluserID,
// avgColumn: avg,
// countColumn: count,
// },
// conflictAlgorithm: ConflictAlgorithm.replace,
// );
// batchCounter++;
// }
// await batch.commit(noResult: true);
}
/// Returns a map of clusterID to (avg embedding, count)
Future<Map<int, (Uint8List, int)>> getAllClusterSummary([
int? minClusterSize,
]) async {
final db = await instance.sqliteAsyncDB;
final db = await instance.asyncDB;
final Map<int, (Uint8List, int)> result = {};
final rows = await db.getAll(
'SELECT * FROM $clusterSummaryTable${minClusterSize != null ? ' WHERE $countColumn >= $minClusterSize' : ''}',
@ -846,9 +915,26 @@ class FaceMLDataDB {
return result;
}
Future<Map<int, (Uint8List, int)>> getClusterToClusterSummary(
Iterable<int> clusterIDs,
) async {
final db = await instance.asyncDB;
final Map<int, (Uint8List, int)> result = {};
final rows = await db.getAll(
'SELECT * FROM $clusterSummaryTable WHERE $clusterIDColumn IN (${clusterIDs.join(",")})',
);
for (final r in rows) {
final id = r[clusterIDColumn] as int;
final avg = r[avgColumn] as Uint8List;
final count = r[countColumn] as int;
result[id] = (avg, count);
}
return result;
}
Future<Map<int, String>> getClusterIDToPersonID() async {
final db = await instance.database;
final List<Map<String, dynamic>> maps = await db.rawQuery(
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $personIdColumn, $clusterIDColumn FROM $clusterPersonTable',
);
final Map<int, String> result = {};
@ -860,43 +946,55 @@ class FaceMLDataDB {
/// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes!
Future<void> dropClustersAndPersonTable({bool faces = false}) async {
final db = await instance.database;
if (faces) {
await db.execute(deleteFacesTable);
await db.execute(createFacesTable);
await db.execute(dropFaceClustersTable);
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex);
}
await db.execute(deletePersonTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropClusterSummaryTable);
await db.execute(dropNotPersonFeedbackTable);
try {
final db = await instance.asyncDB;
if (faces) {
await db.execute(deleteFacesTable);
await db.execute(createFacesTable);
await db.execute(dropFaceClustersTable);
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex);
}
await db.execute(createClusterPersonTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(createClusterSummaryTable);
await db.execute(deletePersonTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropClusterSummaryTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(createClusterPersonTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(createClusterSummaryTable);
} catch (e, s) {
_logger.severe('Error dropping clusters and person table', e, s);
}
}
/// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes!
Future<void> dropFeedbackTables() async {
final db = await instance.database;
try {
final db = await instance.asyncDB;
await db.execute(deletePersonTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(dropClusterSummaryTable);
await db.execute(createClusterPersonTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(createClusterSummaryTable);
// Drop the tables
await db.execute(deletePersonTable);
await db.execute(dropClusterPersonTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(dropClusterSummaryTable);
// Recreate the tables
await db.execute(createClusterPersonTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(createClusterSummaryTable);
} catch (e) {
_logger.severe('Error dropping feedback tables', e);
}
}
Future<void> removeFilesFromPerson(
List<EnteFile> files,
String personID,
) async {
final db = await instance.database;
final faceIdsResult = await db.rawQuery(
final db = await instance.asyncDB;
final faceIdsResult = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
'WHERE $clusterPersonTable.$personIdColumn = ?',
@ -922,8 +1020,8 @@ class FaceMLDataDB {
List<EnteFile> files,
int clusterID,
) async {
final db = await instance.database;
final faceIdsResult = await db.rawQuery(
final db = await instance.asyncDB;
final faceIdsResult = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable '
'WHERE $faceClustersTable.$fcClusterID = ?',
[clusterID],

5
mobile/lib/face/db_fields.dart
View file

@ -16,7 +16,7 @@ const mlVersionColumn = 'ml_version';
const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable (
$fileIDColumn INTEGER NOT NULL,
$faceIDColumn TEXT NOT NULL,
$faceIDColumn TEXT NOT NULL UNIQUE,
$faceDetectionColumn TEXT NOT NULL,
$faceEmbeddingBlob BLOB NOT NULL,
$faceScore REAL NOT NULL,
@ -95,7 +95,8 @@ const notPersonFeedback = 'not_person_feedback';
const createNotPersonFeedbackTable = '''
CREATE TABLE IF NOT EXISTS $notPersonFeedback (
$personIdColumn TEXT NOT NULL,
$clusterIDColumn INTEGER NOT NULL
$clusterIDColumn INTEGER NOT NULL,
PRIMARY KEY($personIdColumn, $clusterIDColumn)
);
''';
const dropNotPersonFeedbackTable = 'DROP TABLE IF EXISTS $notPersonFeedback';

2
mobile/lib/face/model/detection.dart
View file

@ -155,7 +155,7 @@ class Detection {
(nose[0] < min(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) &&
(nose[0] < min(leftMouth[0], rightMouth[0]));
final bool noseStickingOutRight =
(nose[0] > max(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) &&
(nose[0] > max(leftEye[0], rightEye[0]) + 0.5 * eyeDistanceX) &&
(nose[0] > max(leftMouth[0], rightMouth[0]));
return faceIsUpright && (noseStickingOutLeft || noseStickingOutRight);

2
mobile/lib/services/entity_service.dart
View file

@ -61,7 +61,7 @@ class EntityService {
}) async {
final key = await getOrCreateEntityKey(type);
final encryptedKeyData = await CryptoUtil.encryptChaCha(
utf8.encode(plainText) as Uint8List,
utf8.encode(plainText),
key,
);
final String encryptedData =

13
mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart
View file

@ -1,5 +1,18 @@
import 'dart:math' show sqrt;
import "package:ml_linalg/vector.dart";
/// Calculates the cosine distance between two embeddings/vectors using SIMD from ml_linalg
///
/// WARNING: This assumes both vectors are already normalized!
double cosineDistanceSIMD(Vector vector1, Vector vector2) {
if (vector1.length != vector2.length) {
throw ArgumentError('Vectors must be the same length');
}
return 1 - vector1.dot(vector2);
}
/// Calculates the cosine distance between two embeddings/vectors.
///
/// Throws an ArgumentError if the vectors are of different lengths or

32
mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart
View file

@ -69,7 +69,7 @@ class FaceClusteringService {
bool isRunning = false;
static const kRecommendedDistanceThreshold = 0.24;
static const kConservativeDistanceThreshold = 0.06;
static const kConservativeDistanceThreshold = 0.16;
// singleton pattern
FaceClusteringService._privateConstructor();
@ -560,10 +560,10 @@ class FaceClusteringService {
for (int j = i - 1; j >= 0; j--) {
late double distance;
if (sortedFaceInfos[i].vEmbedding != null) {
distance = 1.0 -
sortedFaceInfos[i]
.vEmbedding!
.dot(sortedFaceInfos[j].vEmbedding!);
distance = cosineDistanceSIMD(
sortedFaceInfos[i].vEmbedding!,
sortedFaceInfos[j].vEmbedding!,
);
} else {
distance = cosineDistForNormVectors(
sortedFaceInfos[i].embedding!,
@ -624,7 +624,7 @@ class FaceClusteringService {
}
// analyze the results
FaceClusteringService._analyzeClusterResults(sortedFaceInfos);
// FaceClusteringService._analyzeClusterResults(sortedFaceInfos);
return ClusteringResult(
newFaceIdToCluster: newFaceIdToCluster,
@ -804,8 +804,10 @@ class FaceClusteringService {
double closestDistance = double.infinity;
for (int j = 0; j < totalFaces; j++) {
if (i == j) continue;
final double distance =
1.0 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!);
final double distance = cosineDistanceSIMD(
faceInfos[i].vEmbedding!,
faceInfos[j].vEmbedding!,
);
if (distance < closestDistance) {
closestDistance = distance;
closestIdx = j;
@ -855,10 +857,10 @@ class FaceClusteringService {
for (int i = 0; i < clusterIds.length; i++) {
for (int j = 0; j < clusterIds.length; j++) {
if (i == j) continue;
final double newDistance = 1.0 -
clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1.dot(
clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1,
);
final double newDistance = cosineDistanceSIMD(
clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1,
clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1,
);
if (newDistance < distance) {
distance = newDistance;
clusterIDsToMerge = (clusterIds[i], clusterIds[j]);
@ -959,9 +961,9 @@ class FaceClusteringService {
// Run the DBSCAN clustering
final List<List<int>> clusterOutput = dbscan.run(embeddings);
final List<List<FaceInfo>> clusteredFaceInfos = clusterOutput
.map((cluster) => cluster.map((idx) => faceInfos[idx]).toList())
.toList();
// final List<List<FaceInfo>> clusteredFaceInfos = clusterOutput
// .map((cluster) => cluster.map((idx) => faceInfos[idx]).toList())
// .toList();
final List<List<String>> clusteredFaceIDs = clusterOutput
.map((cluster) => cluster.map((idx) => faceInfos[idx].faceID).toList())
.toList();

7
mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart
View file

@ -1,8 +1,8 @@
import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart';
/// Blur detection threshold
const kLaplacianHardThreshold = 15;
const kLaplacianSoftThreshold = 100;
const kLaplacianHardThreshold = 10;
const kLaplacianSoftThreshold = 50;
const kLaplacianVerySoftThreshold = 200;
/// Default blur value
@ -15,3 +15,6 @@ const kHighQualityFaceScore = 0.90;
/// The minimum score for a face to be detected, regardless of quality. Use [kMinimumQualityFaceScore] for high quality faces.
const kMinFaceDetectionScore = FaceDetectionService.kMinScoreSigmoidThreshold;
/// The minimum cluster size for displaying a cluster in the UI
const kMinimumClusterSizeSearchResult = 20;

21
mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
View file

@ -295,6 +295,7 @@ class FaceMlService {
bool clusterInBuckets = true,
}) async {
_logger.info("`clusterAllImages()` called");
final clusterAllImagesTime = DateTime.now();
try {
// Get a sense of the total number of faces in the database
@ -349,7 +350,7 @@ class FaceMlService {
}
await FaceMLDataDB.instance
.updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster);
.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries!);
_logger.info(
@ -402,13 +403,14 @@ class FaceMlService {
'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB',
);
await FaceMLDataDB.instance
.updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster);
.updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries!);
_logger.info('Done updating FaceIDs with clusterIDs in the DB, in '
'${DateTime.now().difference(clusterDoneTime).inSeconds} seconds');
}
_logger.info('clusterAllImages() finished');
_logger.info('clusterAllImages() finished, in '
'${DateTime.now().difference(clusterAllImagesTime).inSeconds} seconds');
} catch (e, s) {
_logger.severe("`clusterAllImages` failed", e, s);
}
@ -868,7 +870,7 @@ class FaceMlService {
stopwatch.stop();
_logger.info(
"Finished Analyze image (${result.faces.length} faces) with uploadedFileID ${enteFile.uploadedFileID}, in "
"${stopwatch.elapsedMilliseconds} ms",
"${stopwatch.elapsedMilliseconds} ms (including time waiting for inference engine availability)",
);
return result;
@ -964,7 +966,12 @@ class FaceMlService {
switch (typeOfData) {
case FileDataForML.fileData:
final stopwatch = Stopwatch()..start();
final File? file = await getFile(enteFile, isOrigin: true);
File? file;
if (enteFile.fileType == FileType.video) {
file = await getThumbnailForUploadedFile(enteFile);
} else {
file = await getFile(enteFile, isOrigin: true);
}
if (file == null) {
_logger.warning("Could not get file for $enteFile");
imagePath = null;
@ -1292,10 +1299,6 @@ class FaceMlService {
if (!enteFile.isUploaded || enteFile.isOwner == false) {
return true;
}
// Skip if the file is a video
if (enteFile.fileType == FileType.video) {
return true;
}
// I don't know how motionPhotos and livePhotos work, so I'm also just skipping them for now
if (enteFile.fileType == FileType.other) {
return true;

451
mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
View file

@ -1,20 +1,20 @@
import 'dart:developer' as dev;
import "dart:math" show Random;
import "dart:math" show Random, min;
import "package:flutter/foundation.dart";
import "package:logging/logging.dart";
import "package:ml_linalg/linalg.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
// import "package:photos/events/files_updated_event.dart";
// import "package:photos/events/local_photos_updated_event.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import "package:photos/generated/protos/ente/common/vector.pb.dart";
import "package:photos/models/file/file.dart";
import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart';
import "package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/face_ml_result.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/search_service.dart";
@ -24,12 +24,14 @@ class ClusterSuggestion {
final double distancePersonToCluster;
final bool usedOnlyMeanForSuggestion;
final List<EnteFile> filesInCluster;
final List<String> faceIDsInCluster;
ClusterSuggestion(
this.clusterIDToMerge,
this.distancePersonToCluster,
this.usedOnlyMeanForSuggestion,
this.filesInCluster,
this.faceIDsInCluster,
);
}
@ -59,19 +61,27 @@ class ClusterFeedbackService {
bool extremeFilesFirst = true,
}) async {
_logger.info(
'getClusterFilesForPersonID ${kDebugMode ? person.data.name : person.remoteID}',
'getSuggestionForPerson ${kDebugMode ? person.data.name : person.remoteID}',
);
try {
// Get the suggestions for the person using centroids and median
final List<(int, double, bool)> suggestClusterIds =
await _getSuggestionsUsingMedian(person);
final startTime = DateTime.now();
final List<(int, double, bool)> foundSuggestions =
await _getSuggestions(person);
final findSuggestionsTime = DateTime.now();
_logger.info(
'getSuggestionForPerson `_getSuggestions`: Found ${foundSuggestions.length} suggestions in ${findSuggestionsTime.difference(startTime).inMilliseconds} ms',
);
// Get the files for the suggestions
final suggestionClusterIDs = foundSuggestions.map((e) => e.$1).toSet();
final Map<int, Set<int>> fileIdToClusterID =
await FaceMLDataDB.instance.getFileIdToClusterIDSetForCluster(
suggestClusterIds.map((e) => e.$1).toSet(),
suggestionClusterIDs,
);
final clusterIdToFaceIDs =
await FaceMLDataDB.instance.getClusterToFaceIDs(suggestionClusterIDs);
final Map<int, List<EnteFile>> clusterIDToFiles = {};
final allFiles = await SearchService.instance.getAllFiles();
for (final f in allFiles) {
@ -88,25 +98,31 @@ class ClusterFeedbackService {
}
}
final List<ClusterSuggestion> clusterIdAndFiles = [];
for (final clusterSuggestion in suggestClusterIds) {
final List<ClusterSuggestion> finalSuggestions = [];
for (final clusterSuggestion in foundSuggestions) {
if (clusterIDToFiles.containsKey(clusterSuggestion.$1)) {
clusterIdAndFiles.add(
finalSuggestions.add(
ClusterSuggestion(
clusterSuggestion.$1,
clusterSuggestion.$2,
clusterSuggestion.$3,
clusterIDToFiles[clusterSuggestion.$1]!,
clusterIdToFaceIDs[clusterSuggestion.$1]!.toList(),
),
);
}
}
final getFilesTime = DateTime.now();
final sortingStartTime = DateTime.now();
if (extremeFilesFirst) {
await _sortSuggestionsOnDistanceToPerson(person, clusterIdAndFiles);
await _sortSuggestionsOnDistanceToPerson(person, finalSuggestions);
}
_logger.info(
'getSuggestionForPerson post-processing suggestions took ${DateTime.now().difference(findSuggestionsTime).inMilliseconds} ms, of which sorting took ${DateTime.now().difference(sortingStartTime).inMilliseconds} ms and getting files took ${getFilesTime.difference(findSuggestionsTime).inMilliseconds} ms',
);
return clusterIdAndFiles;
return finalSuggestions;
} catch (e, s) {
_logger.severe("Error in getClusterFilesForPersonID", e, s);
rethrow;
@ -228,20 +244,20 @@ class ClusterFeedbackService {
final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID);
final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID);
dev.log(
'existing clusters for ${p.data.name} are $personClusters',
'${p.data.name} has ${personClusters.length} existing clusters',
name: "ClusterFeedbackService",
);
// Get and update the cluster summary to get the avg (centroid) and count
final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start();
final Map<int, List<double>> clusterAvg = await _getUpdateClusterAvg(
final Map<int, Vector> clusterAvg = await _getUpdateClusterAvg(
allClusterIdsToCountMap,
ignoredClusters,
);
watch.log('computed avg for ${clusterAvg.length} clusters');
// Find the actual closest clusters for the person
final Map<int, List<(int, double)>> suggestions = _calcSuggestionsMean(
final List<(int, double)> suggestions = _calcSuggestionsMean(
clusterAvg,
personClusters,
ignoredClusters,
@ -257,21 +273,17 @@ class ClusterFeedbackService {
}
// log suggestions
for (final entry in suggestions.entries) {
dev.log(
' ${entry.value.length} suggestion for ${p.data.name} for cluster ID ${entry.key} are suggestions ${entry.value}}',
name: "ClusterFeedbackService",
);
}
dev.log(
'suggestions for ${p.data.name} for cluster ID ${p.remoteID} are suggestions $suggestions}',
name: "ClusterFeedbackService",
);
for (final suggestionsPerCluster in suggestions.values) {
for (final suggestion in suggestionsPerCluster) {
final clusterID = suggestion.$1;
await PersonService.instance.assignClusterToPerson(
personID: p.remoteID,
clusterID: clusterID,
);
}
for (final suggestion in suggestions) {
final clusterID = suggestion.$1;
await PersonService.instance.assignClusterToPerson(
personID: p.remoteID,
clusterID: clusterID,
);
}
Bus.instance.fire(PeopleChangedEvent());
@ -400,7 +412,7 @@ class ClusterFeedbackService {
final newClusterID = startClusterID + blurValue ~/ 10;
faceIdToCluster[faceID] = newClusterID;
}
await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster);
await FaceMLDataDB.instance.updateFaceIdToClusterId(faceIdToCluster);
Bus.instance.fire(PeopleChangedEvent());
} catch (e, s) {
@ -433,111 +445,89 @@ class ClusterFeedbackService {
return;
}
/// Returns a map of person's clusterID to map of closest clusterID to with disstance
Future<Map<int, List<(int, double)>>> getSuggestionsUsingMean(
PersonEntity p, {
double maxClusterDistance = 0.4,
}) async {
// Get all the cluster data
final faceMlDb = FaceMLDataDB.instance;
final allClusterIdsToCountMap = (await faceMlDb.clusterIdToFaceCount());
final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID);
final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID);
dev.log(
'existing clusters for ${p.data.name} are $personClusters',
name: "ClusterFeedbackService",
);
// Get and update the cluster summary to get the avg (centroid) and count
final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start();
final Map<int, List<double>> clusterAvg = await _getUpdateClusterAvg(
allClusterIdsToCountMap,
ignoredClusters,
);
watch.log('computed avg for ${clusterAvg.length} clusters');
// Find the actual closest clusters for the person
final Map<int, List<(int, double)>> suggestions = _calcSuggestionsMean(
clusterAvg,
personClusters,
ignoredClusters,
maxClusterDistance,
);
// log suggestions
for (final entry in suggestions.entries) {
dev.log(
' ${entry.value.length} suggestion for ${p.data.name} for cluster ID ${entry.key} are suggestions ${entry.value}}',
name: "ClusterFeedbackService",
);
}
return suggestions;
}
/// Returns a list of suggestions. For each suggestion we return a record consisting of the following elements:
/// 1. clusterID: the ID of the cluster
/// 2. distance: the distance between the person's cluster and the suggestion
/// 3. usedMean: whether the suggestion was found using the mean (true) or the median (false)
Future<List<(int, double, bool)>> _getSuggestionsUsingMedian(
Future<List<(int, double, bool)>> _getSuggestions(
PersonEntity p, {
int sampleSize = 50,
double maxMedianDistance = 0.65,
double maxMedianDistance = 0.62,
double goodMedianDistance = 0.55,
double maxMeanDistance = 0.65,
double goodMeanDistance = 0.4,
double goodMeanDistance = 0.50,
}) async {
final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start();
// Get all the cluster data
final faceMlDb = FaceMLDataDB.instance;
// final Map<int, List<(int, double)>> suggestions = {};
final allClusterIdsToCountMap = (await faceMlDb.clusterIdToFaceCount());
final allClusterIdsToCountMap = await faceMlDb.clusterIdToFaceCount();
final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID);
final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID);
dev.log(
'existing clusters for ${p.data.name} are $personClusters',
name: "getSuggestionsUsingMedian",
final personFaceIDs =
await FaceMLDataDB.instance.getFaceIDsForPerson(p.remoteID);
final personFileIDs = personFaceIDs.map(getFileIdFromFaceId).toSet();
w?.log(
'${p.data.name} has ${personClusters.length} existing clusters, getting all database data done',
);
final allClusterIdToFaceIDs =
await FaceMLDataDB.instance.getAllClusterIdToFaceIDs();
w?.log('getAllClusterIdToFaceIDs done');
// Get and update the cluster summary to get the avg (centroid) and count
final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start();
final Map<int, List<double>> clusterAvg = await _getUpdateClusterAvg(
allClusterIdsToCountMap,
ignoredClusters,
);
watch.log('computed avg for ${clusterAvg.length} clusters');
// Find the other cluster candidates based on the mean
final Map<int, List<(int, double)>> suggestionsMean = _calcSuggestionsMean(
clusterAvg,
personClusters,
ignoredClusters,
goodMeanDistance,
);
if (suggestionsMean.isNotEmpty) {
final List<(int, double)> suggestClusterIds = [];
for (final List<(int, double)> suggestion in suggestionsMean.values) {
suggestClusterIds.addAll(suggestion);
// First only do a simple check on the big clusters, if the person does not have small clusters yet
final smallestPersonClusterSize = personClusters
.map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0)
.reduce((value, element) => min(value, element));
final checkSizes = [20, kMinimumClusterSizeSearchResult, 10, 5, 1];
late Map<int, Vector> clusterAvgBigClusters;
final List<(int, double)> suggestionsMean = [];
for (final minimumSize in checkSizes.toSet()) {
// if (smallestPersonClusterSize >= minimumSize) {
clusterAvgBigClusters = await _getUpdateClusterAvg(
allClusterIdsToCountMap,
ignoredClusters,
minClusterSize: minimumSize,
);
w?.log(
'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize',
);
final List<(int, double)> suggestionsMeanBigClusters =
_calcSuggestionsMean(
clusterAvgBigClusters,
personClusters,
ignoredClusters,
goodMeanDistance,
);
w?.log(
'Calculate suggestions using mean for ${clusterAvgBigClusters.length} clusters of min size $minimumSize',
);
for (final suggestion in suggestionsMeanBigClusters) {
// Skip suggestions that have a high overlap with the person's files
final suggestionSet = allClusterIdToFaceIDs[suggestion.$1]!
.map((faceID) => getFileIdFromFaceId(faceID))
.toSet();
final overlap = personFileIDs.intersection(suggestionSet);
if (overlap.isNotEmpty &&
((overlap.length / suggestionSet.length) > 0.5)) {
await FaceMLDataDB.instance.captureNotPersonFeedback(
personID: p.remoteID,
clusterID: suggestion.$1,
);
continue;
}
suggestionsMean.add(suggestion);
}
if (suggestionsMean.isNotEmpty) {
return suggestionsMean
.map((e) => (e.$1, e.$2, true))
.toList(growable: false);
// }
}
suggestClusterIds.sort(
(a, b) => allClusterIdsToCountMap[b.$1]!
.compareTo(allClusterIdsToCountMap[a.$1]!),
);
final suggestClusterIdsSizes = suggestClusterIds
.map((e) => allClusterIdsToCountMap[e.$1]!)
.toList(growable: false);
final suggestClusterIdsDistances =
suggestClusterIds.map((e) => e.$2).toList(growable: false);
_logger.info(
"Already found good suggestions using mean: $suggestClusterIds, with sizes $suggestClusterIdsSizes and distances $suggestClusterIdsDistances",
);
return suggestClusterIds
.map((e) => (e.$1, e.$2, true))
.toList(growable: false);
}
w?.reset();
// Find the other cluster candidates based on the median
final Map<int, List<(int, double)>> moreSuggestionsMean =
_calcSuggestionsMean(
final clusterAvg = clusterAvgBigClusters;
final List<(int, double)> moreSuggestionsMean = _calcSuggestionsMean(
clusterAvg,
personClusters,
ignoredClusters,
@ -549,12 +539,8 @@ class ClusterFeedbackService {
return [];
}
final List<(int, double)> temp = [];
for (final List<(int, double)> suggestion in moreSuggestionsMean.values) {
temp.addAll(suggestion);
}
temp.sort((a, b) => a.$2.compareTo(b.$2));
final otherClusterIdsCandidates = temp
moreSuggestionsMean.sort((a, b) => a.$2.compareTo(b.$2));
final otherClusterIdsCandidates = moreSuggestionsMean
.map(
(e) => e.$1,
)
@ -563,21 +549,26 @@ class ClusterFeedbackService {
"Found potential suggestions from loose mean for median test: $otherClusterIdsCandidates",
);
watch.logAndReset("Starting median test");
w?.logAndReset("Starting median test");
// Take the embeddings from the person's clusters in one big list and sample from it
final List<Uint8List> personEmbeddingsProto = [];
for (final clusterID in personClusters) {
final Iterable<Uint8List> embedings =
final Iterable<Uint8List> embeddings =
await FaceMLDataDB.instance.getFaceEmbeddingsForCluster(clusterID);
personEmbeddingsProto.addAll(embedings);
personEmbeddingsProto.addAll(embeddings);
}
final List<Uint8List> sampledEmbeddingsProto =
_randomSampleWithoutReplacement(
personEmbeddingsProto,
sampleSize,
);
final List<List<double>> sampledEmbeddings = sampledEmbeddingsProto
.map((embedding) => EVector.fromBuffer(embedding).values)
final List<Vector> sampledEmbeddings = sampledEmbeddingsProto
.map(
(embedding) => Vector.fromList(
EVector.fromBuffer(embedding).values,
dtype: DType.float32,
),
)
.toList(growable: false);
// Find the actual closest clusters for the person using median
@ -593,16 +584,20 @@ class ClusterFeedbackService {
otherEmbeddingsProto,
sampleSize,
);
final List<List<double>> sampledOtherEmbeddings =
sampledOtherEmbeddingsProto
.map((embedding) => EVector.fromBuffer(embedding).values)
.toList(growable: false);
final List<Vector> sampledOtherEmbeddings = sampledOtherEmbeddingsProto
.map(
(embedding) => Vector.fromList(
EVector.fromBuffer(embedding).values,
dtype: DType.float32,
),
)
.toList(growable: false);
// Calculate distances and find the median
final List<double> distances = [];
for (final otherEmbedding in sampledOtherEmbeddings) {
for (final embedding in sampledEmbeddings) {
distances.add(cosineDistForNormVectors(embedding, otherEmbedding));
distances.add(cosineDistanceSIMD(embedding, otherEmbedding));
}
}
distances.sort();
@ -616,7 +611,7 @@ class ClusterFeedbackService {
}
}
}
watch.log("Finished median test");
w?.log("Finished median test");
if (suggestionsMedian.isEmpty) {
_logger.info("No suggestions found using median");
return [];
@ -648,23 +643,29 @@ class ClusterFeedbackService {
return finalSuggestionsMedian;
}
Future<Map<int, List<double>>> _getUpdateClusterAvg(
Future<Map<int, Vector>> _getUpdateClusterAvg(
Map<int, int> allClusterIdsToCountMap,
Set<int> ignoredClusters, {
int minClusterSize = 1,
int maxClusterInCurrentRun = 500,
int maxEmbeddingToRead = 10000,
}) async {
final w = (kDebugMode ? EnteWatch('_getUpdateClusterAvg') : null)?..start();
final startTime = DateTime.now();
final faceMlDb = FaceMLDataDB.instance;
_logger.info(
'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters, minClusterSize $minClusterSize, maxClusterInCurrentRun $maxClusterInCurrentRun',
);
final Map<int, (Uint8List, int)> clusterToSummary =
await faceMlDb.getAllClusterSummary();
await faceMlDb.getAllClusterSummary(minClusterSize);
final Map<int, (Uint8List, int)> updatesForClusterSummary = {};
final Map<int, List<double>> clusterAvg = {};
final Map<int, Vector> clusterAvg = {};
w?.log(
'getUpdateClusterAvg database call for getAllClusterSummary',
);
final allClusterIds = allClusterIdsToCountMap.keys.toSet();
int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0;
@ -676,7 +677,10 @@ class ClusterFeedbackService {
}
if (clusterToSummary[id]?.$2 == allClusterIdsToCountMap[id]) {
allClusterIds.remove(id);
clusterAvg[id] = EVector.fromBuffer(clusterToSummary[id]!.$1).values;
clusterAvg[id] = Vector.fromList(
EVector.fromBuffer(clusterToSummary[id]!.$1).values,
dtype: DType.float32,
);
alreadyUpdatedClustersCnt++;
}
if (allClusterIdsToCountMap[id]! < minClusterSize) {
@ -684,9 +688,20 @@ class ClusterFeedbackService {
smallerClustersCnt++;
}
}
w?.log(
'serialization of embeddings',
);
_logger.info(
'Ignored $ignoredClustersCnt clusters, already updated $alreadyUpdatedClustersCnt clusters, $smallerClustersCnt clusters are smaller than $minClusterSize',
);
if (allClusterIds.isEmpty) {
_logger.info(
'No clusters to update, getUpdateClusterAvg done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
);
return clusterAvg;
}
// get clusterIDs sorted by count in descending order
final sortedClusterIDs = allClusterIds.toList();
sortedClusterIDs.sort(
@ -694,12 +709,7 @@ class ClusterFeedbackService {
allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!),
);
int indexedInCurrentRun = 0;
final EnteWatch? w = kDebugMode ? EnteWatch("computeAvg") : null;
w?.start();
w?.log(
'reading embeddings for $maxClusterInCurrentRun or ${sortedClusterIDs.length} clusters',
);
w?.reset();
int currentPendingRead = 0;
final List<int> clusterIdsToRead = [];
@ -730,19 +740,17 @@ class ClusterFeedbackService {
);
for (final clusterID in clusterEmbeddings.keys) {
late List<double> avg;
final Iterable<Uint8List> embedings = clusterEmbeddings[clusterID]!;
final List<double> sum = List.filled(192, 0);
for (final embedding in embedings) {
final data = EVector.fromBuffer(embedding).values;
for (int i = 0; i < sum.length; i++) {
sum[i] += data[i];
}
}
avg = sum.map((e) => e / embedings.length).toList();
final avgEmbeedingBuffer = EVector(values: avg).writeToBuffer();
final Iterable<Uint8List> embeddings = clusterEmbeddings[clusterID]!;
final Iterable<Vector> vectors = embeddings.map(
(e) => Vector.fromList(
EVector.fromBuffer(e).values,
dtype: DType.float32,
),
);
final avg = vectors.reduce((a, b) => a + b) / vectors.length;
final avgEmbeddingBuffer = EVector(values: avg).writeToBuffer();
updatesForClusterSummary[clusterID] =
(avgEmbeedingBuffer, embedings.length);
(avgEmbeddingBuffer, embeddings.length);
// store the intermediate updates
indexedInCurrentRun++;
if (updatesForClusterSummary.length > 100) {
@ -760,26 +768,31 @@ class ClusterFeedbackService {
await faceMlDb.clusterSummaryUpdate(updatesForClusterSummary);
}
w?.logAndReset('done computing avg ');
_logger.info('end getUpdateClusterAvg for ${clusterAvg.length} clusters');
_logger.info(
'end getUpdateClusterAvg for ${clusterAvg.length} clusters, done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
);
return clusterAvg;
}
/// Returns a map of person's clusterID to map of closest clusterID to with disstance
Map<int, List<(int, double)>> _calcSuggestionsMean(
Map<int, List<double>> clusterAvg,
List<(int, double)> _calcSuggestionsMean(
Map<int, Vector> clusterAvg,
Set<int> personClusters,
Set<int> ignoredClusters,
double maxClusterDistance,
) {
double maxClusterDistance, {
Map<int, int>? allClusterIdsToCountMap,
}) {
final Map<int, List<(int, double)>> suggestions = {};
int suggestionCount = 0;
final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start();
for (final otherClusterID in clusterAvg.keys) {
// ignore the cluster that belong to the person or is ignored
if (personClusters.contains(otherClusterID) ||
ignoredClusters.contains(otherClusterID)) {
continue;
}
final otherAvg = clusterAvg[otherClusterID]!;
final Vector otherAvg = clusterAvg[otherClusterID]!;
int? nearestPersonCluster;
double? minDistance;
for (final personCluster in personClusters) {
@ -787,8 +800,8 @@ class ClusterFeedbackService {
_logger.info('no avg for cluster $personCluster');
continue;
}
final avg = clusterAvg[personCluster]!;
final distance = cosineDistForNormVectors(avg, otherAvg);
final Vector avg = clusterAvg[personCluster]!;
final distance = cosineDistanceSIMD(avg, otherAvg);
if (distance < maxClusterDistance) {
if (minDistance == null || distance < minDistance) {
minDistance = distance;
@ -800,13 +813,39 @@ class ClusterFeedbackService {
suggestions
.putIfAbsent(nearestPersonCluster, () => [])
.add((otherClusterID, minDistance));
suggestionCount++;
}
if (suggestionCount >= 2000) {
break;
}
}
for (final entry in suggestions.entries) {
entry.value.sort((a, b) => a.$1.compareTo(b.$1));
}
w?.log('calculation inside calcSuggestionsMean');
return suggestions;
if (suggestions.isNotEmpty) {
final List<(int, double)> suggestClusterIds = [];
for (final List<(int, double)> suggestion in suggestions.values) {
suggestClusterIds.addAll(suggestion);
}
suggestClusterIds.sort(
(a, b) => a.$2.compareTo(b.$2),
); // sort by distance
// List<int>? suggestClusterIdsSizes;
// if (allClusterIdsToCountMap != null) {
// suggestClusterIdsSizes = suggestClusterIds
// .map((e) => allClusterIdsToCountMap[e.$1]!)
// .toList(growable: false);
// }
// final suggestClusterIdsDistances =
// suggestClusterIds.map((e) => e.$2).toList(growable: false);
_logger.info(
"Already found ${suggestClusterIds.length} good suggestions using mean",
);
return suggestClusterIds.sublist(0, min(suggestClusterIds.length, 20));
} else {
_logger.info("No suggestions found using mean");
return <(int, double)>[];
}
}
List<T> _randomSampleWithoutReplacement<T>(
@ -841,56 +880,88 @@ class ClusterFeedbackService {
Future<void> _sortSuggestionsOnDistanceToPerson(
PersonEntity person,
List<ClusterSuggestion> suggestions,
) async {
List<ClusterSuggestion> suggestions, {
bool onlySortBigSuggestions = true,
}) async {
if (suggestions.isEmpty) {
debugPrint('No suggestions to sort');
return;
}
if (onlySortBigSuggestions) {
final bigSuggestions = suggestions
.where(
(s) => s.filesInCluster.length > kMinimumClusterSizeSearchResult,
)
.toList();
if (bigSuggestions.isEmpty) {
debugPrint('No big suggestions to sort');
return;
}
}
final startTime = DateTime.now();
final faceMlDb = FaceMLDataDB.instance;
// Get the cluster averages for the person's clusters and the suggestions' clusters
final Map<int, (Uint8List, int)> clusterToSummary =
await faceMlDb.getAllClusterSummary();
final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID);
final Map<int, (Uint8List, int)> personClusterToSummary =
await faceMlDb.getClusterToClusterSummary(personClusters);
final clusterSummaryCallTime = DateTime.now();
// Calculate the avg embedding of the person
final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID);
final w = (kDebugMode ? EnteWatch('sortSuggestions') : null)?..start();
final personEmbeddingsCount = personClusters
.map((e) => clusterToSummary[e]!.$2)
.map((e) => personClusterToSummary[e]!.$2)
.reduce((a, b) => a + b);
final List<double> personAvg = List.filled(192, 0);
Vector personAvg = Vector.filled(192, 0);
for (final personClusterID in personClusters) {
final personClusterBlob = clusterToSummary[personClusterID]!.$1;
final personClusterAvg = EVector.fromBuffer(personClusterBlob).values;
final personClusterBlob = personClusterToSummary[personClusterID]!.$1;
final personClusterAvg = Vector.fromList(
EVector.fromBuffer(personClusterBlob).values,
dtype: DType.float32,
);
final clusterWeight =
clusterToSummary[personClusterID]!.$2 / personEmbeddingsCount;
for (int i = 0; i < personClusterAvg.length; i++) {
personAvg[i] += personClusterAvg[i] *
clusterWeight; // Weighted sum of the cluster averages
}
personClusterToSummary[personClusterID]!.$2 / personEmbeddingsCount;
personAvg += personClusterAvg * clusterWeight;
}
w?.log('calculated person avg');
// Sort the suggestions based on the distance to the person
for (final suggestion in suggestions) {
if (onlySortBigSuggestions) {
if (suggestion.filesInCluster.length <= 8) {
continue;
}
}
final clusterID = suggestion.clusterIDToMerge;
final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFile(
suggestion.filesInCluster.map((e) => e.uploadedFileID!).toList(),
final faceIDs = suggestion.faceIDsInCluster;
final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFaces(
faceIDs,
);
final faceIdToVectorMap = faceIdToEmbeddingMap.map(
(key, value) => MapEntry(
key,
Vector.fromList(
EVector.fromBuffer(value).values,
dtype: DType.float32,
),
),
);
w?.log(
'got ${faceIdToEmbeddingMap.values.length} embeddings for ${suggestion.filesInCluster.length} files for cluster $clusterID',
);
final fileIdToDistanceMap = {};
for (final entry in faceIdToEmbeddingMap.entries) {
for (final entry in faceIdToVectorMap.entries) {
fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] =
cosineDistForNormVectors(
personAvg,
EVector.fromBuffer(entry.value).values,
);
cosineDistanceSIMD(personAvg, entry.value);
}
w?.log('calculated distances for cluster $clusterID');
suggestion.filesInCluster.sort((b, a) {
//todo: review with @laurens, added this to avoid null safety issue
final double distanceA = fileIdToDistanceMap[a.uploadedFileID!] ?? -1;
final double distanceB = fileIdToDistanceMap[b.uploadedFileID!] ?? -1;
return distanceA.compareTo(distanceB);
});
w?.log('sorted files for cluster $clusterID');
debugPrint(
"[${_logger.name}] Sorted suggestions for cluster $clusterID based on distance to person: ${suggestion.filesInCluster.map((e) => fileIdToDistanceMap[e.uploadedFileID]).toList()}",
@ -899,7 +970,7 @@ class ClusterFeedbackService {
final endTime = DateTime.now();
_logger.info(
"Sorting suggestions based on distance to person took ${endTime.difference(startTime).inMilliseconds} ms for ${suggestions.length} suggestions",
"Sorting suggestions based on distance to person took ${endTime.difference(startTime).inMilliseconds} ms for ${suggestions.length} suggestions, of which ${clusterSummaryCallTime.difference(startTime).inMilliseconds} ms was spent on the cluster summary call",
);
}
}

13
mobile/lib/services/machine_learning/face_ml/person/person_service.dart
View file

@ -1,3 +1,4 @@
import "dart:async" show unawaited;
import "dart:convert";
import "package:flutter/foundation.dart";
@ -102,10 +103,12 @@ class PersonService {
faces: faceIds.toSet(),
);
personData.assigned!.add(clusterInfo);
await entityService.addOrUpdate(
EntityType.person,
json.encode(personData.toJson()),
id: personID,
unawaited(
entityService.addOrUpdate(
EntityType.person,
json.encode(personData.toJson()),
id: personID,
),
);
await faceMLDataDB.assignClusterToPerson(
personID: personID,
@ -190,7 +193,7 @@ class PersonService {
}
logger.info("Storing feedback for ${faceIdToClusterID.length} faces");
await faceMLDataDB.updateClusterIdToFaceId(faceIdToClusterID);
await faceMLDataDB.updateFaceIdToClusterId(faceIdToClusterID);
await faceMLDataDB.bulkAssignClusterToPersonID(clusterToPersonID);
}

3
mobile/lib/services/search_service.dart
View file

@ -28,6 +28,7 @@ import "package:photos/models/search/search_constants.dart";
import "package:photos/models/search/search_types.dart";
import 'package:photos/services/collections_service.dart';
import "package:photos/services/location_service.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart';
import "package:photos/states/location_screen_state.dart";
@ -824,7 +825,7 @@ class SearchService {
"Cluster $clusterId should not have person id ${clusterIDToPersonID[clusterId]}",
);
}
if (files.length < 20 && sortedClusterIds.length > 3) {
if (files.length < kMinimumClusterSizeSearchResult && sortedClusterIds.length > 3) {
continue;
}
facesResult.add(

68
mobile/lib/ui/settings/debug/face_debug_section_widget.dart
View file

@ -8,7 +8,6 @@ import "package:photos/events/people_changed_event.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart";
import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart';
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/theme/ente_theme.dart';
import 'package:photos/ui/components/captioned_text_widget.dart';
@ -217,9 +216,14 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
await FaceMLDataDB.instance.dropFeedbackTables();
Bus.instance.fire(PeopleChangedEvent());
showShortToast(context, "Done");
try {
await FaceMLDataDB.instance.dropFeedbackTables();
Bus.instance.fire(PeopleChangedEvent());
showShortToast(context, "Done");
} catch (e, s) {
_logger.warning('reset feedback failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
@ -284,34 +288,34 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
);
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Rank blurs",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
await showChoiceDialog(
context,
title: "Are you sure?",
body:
"This will delete all clusters and put blurry faces in separate clusters per ten points.",
firstButtonLabel: "Yes, confirm",
firstButtonOnTap: () async {
try {
await ClusterFeedbackService.instance
.createFakeClustersByBlurValue();
showShortToast(context, "Done");
} catch (e, s) {
_logger.warning('Failed to rank faces on blur values ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
);
},
),
// sectionOptionSpacing,
// MenuItemWidget(
// captionedTextWidget: const CaptionedTextWidget(
// title: "Rank blurs",
// ),
// pressedColor: getEnteColorScheme(context).fillFaint,
// trailingIcon: Icons.chevron_right_outlined,
// trailingIconIsMuted: true,
// onTap: () async {
// await showChoiceDialog(
// context,
// title: "Are you sure?",
// body:
// "This will delete all clusters and put blurry faces in separate clusters per ten points.",
// firstButtonLabel: "Yes, confirm",
// firstButtonOnTap: () async {
// try {
// await ClusterFeedbackService.instance
// .createFakeClustersByBlurValue();
// showShortToast(context, "Done");
// } catch (e, s) {
// _logger.warning('Failed to rank faces on blur values ', e, s);
// await showGenericErrorDialog(context: context, error: e);
// }
// },
// );
// },
// ),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(

167
mobile/lib/ui/viewer/file_details/face_widget.dart
View file

@ -1,10 +1,10 @@
import "dart:developer" show log;
import "dart:io" show Platform;
import "dart:typed_data";
import "package:flutter/cupertino.dart";
import "package:flutter/foundation.dart" show kDebugMode;
import "package:flutter/material.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart";
import "package:photos/face/model/face.dart";
import "package:photos/face/model/person.dart";
@ -21,6 +21,8 @@ import "package:photos/utils/face/face_box_crop.dart";
import "package:photos/utils/thumbnail_util.dart";
// import "package:photos/utils/toast_util.dart";
const useGeneratedFaceCrops = true;
class FaceWidget extends StatefulWidget {
final EnteFile file;
final Face face;
@ -48,12 +50,13 @@ class _FaceWidgetState extends State<FaceWidget> {
@override
Widget build(BuildContext context) {
if (Platform.isIOS) {
if (useGeneratedFaceCrops) {
return FutureBuilder<Uint8List?>(
future: getFaceCrop(),
builder: (context, snapshot) {
if (snapshot.hasData) {
final ImageProvider imageProvider = MemoryImage(snapshot.data!);
return GestureDetector(
onTap: () async {
if (widget.editMode) return;
@ -63,7 +66,50 @@ class _FaceWidgetState extends State<FaceWidget> {
name: "FaceWidget",
);
if (widget.person == null && widget.clusterID == null) {
return;
// Get faceID and double check that it doesn't belong to an existing clusterID. If it does, push that cluster page
final w = (kDebugMode ? EnteWatch('FaceWidget') : null)
?..start();
final existingClusterID = await FaceMLDataDB.instance
.getClusterIDForFaceID(widget.face.faceID);
w?.log('getting existing clusterID for faceID');
if (existingClusterID != null) {
final fileIdsToClusterIds =
await FaceMLDataDB.instance.getFileIdToClusterIds();
final files = await SearchService.instance.getAllFiles();
final clusterFiles = files
.where(
(file) =>
fileIdsToClusterIds[file.uploadedFileID]
?.contains(existingClusterID) ??
false,
)
.toList();
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
clusterFiles,
clusterID: existingClusterID,
),
),
);
}
// Create new clusterID for the faceID and update DB to assign the faceID to the new clusterID
final int newClusterID =
DateTime.now().microsecondsSinceEpoch;
await FaceMLDataDB.instance.updateFaceIdToClusterId(
{widget.face.faceID: newClusterID},
);
// Push page for the new cluster
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
[widget.file],
clusterID: newClusterID,
),
),
);
}
if (widget.person != null) {
await Navigator.of(context).push(
@ -228,7 +274,49 @@ class _FaceWidgetState extends State<FaceWidget> {
name: "FaceWidget",
);
if (widget.person == null && widget.clusterID == null) {
return;
// Get faceID and double check that it doesn't belong to an existing clusterID. If it does, push that cluster page
final w = (kDebugMode ? EnteWatch('FaceWidget') : null)
?..start();
final existingClusterID = await FaceMLDataDB.instance
.getClusterIDForFaceID(widget.face.faceID);
w?.log('getting existing clusterID for faceID');
if (existingClusterID != null) {
final fileIdsToClusterIds =
await FaceMLDataDB.instance.getFileIdToClusterIds();
final files = await SearchService.instance.getAllFiles();
final clusterFiles = files
.where(
(file) =>
fileIdsToClusterIds[file.uploadedFileID]
?.contains(existingClusterID) ??
false,
)
.toList();
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
clusterFiles,
clusterID: existingClusterID,
),
),
);
}
// Create new clusterID for the faceID and update DB to assign the faceID to the new clusterID
final int newClusterID = DateTime.now().microsecondsSinceEpoch;
await FaceMLDataDB.instance.updateFaceIdToClusterId(
{widget.face.faceID: newClusterID},
);
// Push page for the new cluster
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
[widget.file],
clusterID: newClusterID,
),
),
);
}
if (widget.person != null) {
await Navigator.of(context).push(
@ -262,33 +350,56 @@ class _FaceWidgetState extends State<FaceWidget> {
},
child: Column(
children: [
Container(
height: 60,
width: 60,
decoration: ShapeDecoration(
shape: RoundedRectangleBorder(
borderRadius:
const BorderRadius.all(Radius.elliptical(16, 12)),
side: widget.highlight
? BorderSide(
color: getEnteColorScheme(context).primary700,
width: 2.0,
)
: BorderSide.none,
),
),
child: ClipRRect(
borderRadius:
const BorderRadius.all(Radius.elliptical(16, 12)),
child: SizedBox(
width: 60,
Stack(
children: [
Container(
height: 60,
child: CroppedFaceImageView(
enteFile: widget.file,
face: widget.face,
width: 60,
decoration: ShapeDecoration(
shape: RoundedRectangleBorder(
borderRadius: const BorderRadius.all(
Radius.elliptical(16, 12),
),
side: widget.highlight
? BorderSide(
color: getEnteColorScheme(context).primary700,
width: 1.0,
)
: BorderSide.none,
),
),
child: ClipRRect(
borderRadius:
const BorderRadius.all(Radius.elliptical(16, 12)),
child: SizedBox(
width: 60,
height: 60,
child: CroppedFaceImageView(
enteFile: widget.file,
face: widget.face,
),
),
),
),
),
// TODO: the edges of the green line are still not properly rounded around ClipRRect
if (widget.editMode)
Positioned(
right: 0,
top: 0,
child: GestureDetector(
onTap: _cornerIconPressed,
child: isJustRemoved
? const Icon(
CupertinoIcons.add_circled_solid,
color: Colors.green,
)
: const Icon(
Icons.cancel,
color: Colors.red,
),
),
),
],
),
const SizedBox(height: 8),
if (widget.person != null)

30
mobile/lib/ui/viewer/file_details/faces_item_widget.dart
View file

@ -71,9 +71,9 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
];
}
// Remove faces with low scores and blurry faces
// Remove faces with low scores
if (!kDebugMode) {
faces.removeWhere((face) => (face.isBlurry || face.score < 0.75));
faces.removeWhere((face) => (face.score < 0.75));
}
if (faces.isEmpty) {
@ -85,9 +85,6 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
];
}
// Sort the faces by score in descending order, so that the highest scoring face is first.
faces.sort((Face a, Face b) => b.score.compareTo(a.score));
// TODO: add deduplication of faces of same person
final faceIdsToClusterIds = await FaceMLDataDB.instance
.getFaceIdsToClusterIds(faces.map((face) => face.faceID));
@ -96,6 +93,29 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
final clusterIDToPerson =
await FaceMLDataDB.instance.getClusterIDToPersonID();
// Sort faces by name and score
final faceIdToPersonID = <String, String>{};
for (final face in faces) {
final clusterID = faceIdsToClusterIds[face.faceID];
if (clusterID != null) {
final personID = clusterIDToPerson[clusterID];
if (personID != null) {
faceIdToPersonID[face.faceID] = personID;
}
}
}
faces.sort((Face a, Face b) {
final aPersonID = faceIdToPersonID[a.faceID];
final bPersonID = faceIdToPersonID[b.faceID];
if (aPersonID != null && bPersonID == null) {
return -1;
} else if (aPersonID == null && bPersonID != null) {
return 1;
} else {
return b.score.compareTo(a.score);
}
});
final lastViewedClusterID = ClusterFeedbackService.lastViewedClusterID;
final faceWidgets = <FaceWidget>[];

16
mobile/lib/ui/viewer/people/cluster_app_bar.dart
View file

@ -207,14 +207,14 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
if (embedding.key == otherEmbedding.key) {
continue;
}
final distance64 = 1.0 -
Vector.fromList(embedding.value, dtype: DType.float64).dot(
Vector.fromList(otherEmbedding.value, dtype: DType.float64),
);
final distance32 = 1.0 -
Vector.fromList(embedding.value, dtype: DType.float32).dot(
Vector.fromList(otherEmbedding.value, dtype: DType.float32),
);
final distance64 = cosineDistanceSIMD(
Vector.fromList(embedding.value, dtype: DType.float64),
Vector.fromList(otherEmbedding.value, dtype: DType.float64),
);
final distance32 = cosineDistanceSIMD(
Vector.fromList(embedding.value, dtype: DType.float32),
Vector.fromList(otherEmbedding.value, dtype: DType.float32),
);
final distance = cosineDistForNormVectors(
embedding.value,
otherEmbedding.value,

26
mobile/lib/ui/viewer/people/cropped_face_image_view.dart
View file

@ -4,8 +4,10 @@ import "dart:io" show File;
import 'package:flutter/material.dart';
import "package:photos/face/model/face.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/file/file_type.dart";
import "package:photos/ui/viewer/file/thumbnail_widget.dart";
import "package:photos/utils/file_util.dart";
import "package:photos/utils/thumbnail_util.dart";
class CroppedFaceInfo {
final Image image;
@ -38,7 +40,8 @@ class CroppedFaceImageView extends StatelessWidget {
builder: (context, snapshot) {
if (snapshot.hasData) {
return LayoutBuilder(
builder: (BuildContext context, BoxConstraints constraints) {
builder: ((context, constraints) {
final double imageAspectRatio = enteFile.width / enteFile.height;
final Image image = snapshot.data!;
final double viewWidth = constraints.maxWidth;
@ -51,14 +54,13 @@ class CroppedFaceImageView extends StatelessWidget {
final double relativeFaceCenterY =
faceBox.yMin + faceBox.height / 2;
const double desiredFaceHeightRelativeToWidget = 1 / 2;
const double desiredFaceHeightRelativeToWidget = 8 / 10;
final double scale =
(1 / faceBox.height) * desiredFaceHeightRelativeToWidget;
final double widgetCenterX = viewWidth / 2;
final double widgetCenterY = viewHeight / 2;
final double imageAspectRatio = enteFile.width / enteFile.height;
final double widgetAspectRatio = viewWidth / viewHeight;
final double imageToWidgetRatio =
imageAspectRatio / widgetAspectRatio;
@ -68,16 +70,15 @@ class CroppedFaceImageView extends StatelessWidget {
double offsetY =
(widgetCenterY - relativeFaceCenterY * viewHeight) * scale;
if (imageAspectRatio > widgetAspectRatio) {
if (imageAspectRatio < widgetAspectRatio) {
// Landscape Image: Adjust offsetX more conservatively
offsetX = offsetX * imageToWidgetRatio;
} else {
// Portrait Image: Adjust offsetY more conservatively
offsetY = offsetY / imageToWidgetRatio;
}
return ClipRect(
clipBehavior: Clip.antiAlias,
return ClipRRect(
borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)),
child: Transform.translate(
offset: Offset(
offsetX,
@ -89,7 +90,7 @@ class CroppedFaceImageView extends StatelessWidget {
),
),
);
},
}),
);
} else {
if (snapshot.hasError) {
@ -104,13 +105,18 @@ class CroppedFaceImageView extends StatelessWidget {
}
Future<Image?> getImage() async {
final File? ioFile = await getFile(enteFile);
final File? ioFile;
if (enteFile.fileType == FileType.video) {
ioFile = await getThumbnailForUploadedFile(enteFile);
} else {
ioFile = await getFile(enteFile);
}
if (ioFile == null) {
return null;
}
final imageData = await ioFile.readAsBytes();
final image = Image.memory(imageData, fit: BoxFit.cover);
final image = Image.memory(imageData, fit: BoxFit.contain);
return image;
}

82
mobile/lib/ui/viewer/people/person_cluster_suggestion.dart
View file

@ -1,3 +1,4 @@
import "dart:async" show StreamSubscription, unawaited;
import "dart:math";
import "package:flutter/foundation.dart" show kDebugMode;
@ -29,16 +30,25 @@ class PersonReviewClusterSuggestion extends StatefulWidget {
class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
int currentSuggestionIndex = 0;
bool fetch = true;
Key futureBuilderKey = UniqueKey();
// Declare a variable for the future
late Future<List<ClusterSuggestion>> futureClusterSuggestions;
late StreamSubscription<PeopleChangedEvent> _peopleChangedEvent;
@override
void initState() {
super.initState();
// Initialize the future in initState
_fetchClusterSuggestions();
if (fetch) _fetchClusterSuggestions();
fetch = true;
}
@override
void dispose() {
_peopleChangedEvent.cancel();
super.dispose();
}
@override
@ -61,12 +71,27 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
),
);
}
final numberOfDifferentSuggestions = snapshot.data!.length;
final currentSuggestion = snapshot.data![currentSuggestionIndex];
final allSuggestions = snapshot.data!;
final numberOfDifferentSuggestions = allSuggestions.length;
final currentSuggestion = allSuggestions[currentSuggestionIndex];
final int clusterID = currentSuggestion.clusterIDToMerge;
final double distance = currentSuggestion.distancePersonToCluster;
final bool usingMean = currentSuggestion.usedOnlyMeanForSuggestion;
final List<EnteFile> files = currentSuggestion.filesInCluster;
_peopleChangedEvent =
Bus.instance.on<PeopleChangedEvent>().listen((event) {
if (event.type == PeopleEventType.removedFilesFromCluster &&
(event.source == clusterID.toString())) {
for (var updatedFile in event.relevantFiles!) {
files.remove(updatedFile);
}
fetch = false;
setState(() {});
}
});
return InkWell(
onTap: () {
Navigator.of(context).push(
@ -90,6 +115,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
usingMean,
files,
numberOfDifferentSuggestions,
allSuggestions,
),
),
);
@ -116,20 +142,25 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
clusterID: clusterID,
);
Bus.instance.fire(PeopleChangedEvent());
// Increment the suggestion index
if (mounted) {
setState(() => currentSuggestionIndex++);
}
// Check if we need to fetch new data
if (currentSuggestionIndex >= (numberOfSuggestions)) {
setState(() {
currentSuggestionIndex = 0;
futureBuilderKey = UniqueKey(); // Reset to trigger FutureBuilder
_fetchClusterSuggestions();
});
}
} else {
await FaceMLDataDB.instance.captureNotPersonFeedback(
personID: widget.person.remoteID,
clusterID: clusterID,
);
}
// Increment the suggestion index
if (mounted) {
setState(() => currentSuggestionIndex++);
}
// Check if we need to fetch new data
if (currentSuggestionIndex >= (numberOfSuggestions)) {
// Recalculate the suggestions when a suggestion is rejected
setState(() {
currentSuggestionIndex = 0;
futureBuilderKey = UniqueKey(); // Reset to trigger FutureBuilder
@ -150,9 +181,10 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
bool usingMean,
List<EnteFile> files,
int numberOfSuggestions,
List<ClusterSuggestion> allSuggestions,
) {
return Column(
key: ValueKey("cluster_id-$clusterID"),
final widgetToReturn = Column(
key: ValueKey("cluster_id-$clusterID-files-${files.length}"),
children: <Widget>[
if (kDebugMode)
Text(
@ -228,6 +260,28 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
),
],
);
// Precompute face thumbnails for next suggestions, in case there are
const precompute = 6;
const maxComputations = 10;
int compCount = 0;
if (allSuggestions.length > currentSuggestionIndex + 1) {
for (final suggestion in allSuggestions.sublist(
currentSuggestionIndex + 1,
min(allSuggestions.length, currentSuggestionIndex + precompute),
)) {
final files = suggestion.filesInCluster;
final clusterID = suggestion.clusterIDToMerge;
for (final file in files.sublist(0, min(files.length, 8))) {
unawaited(PersonFaceWidget.precomputeFaceCrops(file, clusterID));
compCount++;
if (compCount >= maxComputations) {
break;
}
}
}
}
return widgetToReturn;
}
List<Widget> _buildThumbnailWidgets(

60
mobile/lib/ui/viewer/search/result/person_face_widget.dart
View file

@ -1,5 +1,5 @@
import "dart:developer";
import "dart:io";
// import "dart:io";
import "dart:typed_data";
import 'package:flutter/widgets.dart';
@ -10,6 +10,7 @@ import "package:photos/face/model/person.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/ui/viewer/file/thumbnail_widget.dart';
import "package:photos/ui/viewer/file_details/face_widget.dart";
import "package:photos/ui/viewer/people/cropped_face_image_view.dart";
import "package:photos/utils/face/face_box_crop.dart";
import "package:photos/utils/thumbnail_util.dart";
@ -32,9 +33,64 @@ class PersonFaceWidget extends StatelessWidget {
),
super(key: key);
static Future<void> precomputeFaceCrops(file, clusterID) async {
try {
final Face? face = await FaceMLDataDB.instance.getCoverFaceForPerson(
recentFileID: file.uploadedFileID!,
clusterID: clusterID,
);
if (face == null) {
debugPrint(
"No cover face for cluster $clusterID and recentFile ${file.uploadedFileID}",
);
return;
}
final Uint8List? cachedFace = faceCropCache.get(face.faceID);
if (cachedFace != null) {
return;
}
final faceCropCacheFile = cachedFaceCropPath(face.faceID);
if ((await faceCropCacheFile.exists())) {
final data = await faceCropCacheFile.readAsBytes();
faceCropCache.put(face.faceID, data);
return;
}
EnteFile? fileForFaceCrop = file;
if (face.fileID != file.uploadedFileID!) {
fileForFaceCrop =
await FilesDB.instance.getAnyUploadedFile(face.fileID);
}
if (fileForFaceCrop == null) {
return;
}
final result = await pool.withResource(
() async => await getFaceCrops(
fileForFaceCrop!,
{
face.faceID: face.detection.box,
},
),
);
final Uint8List? computedCrop = result?[face.faceID];
if (computedCrop != null) {
faceCropCache.put(face.faceID, computedCrop);
faceCropCacheFile.writeAsBytes(computedCrop).ignore();
}
return;
} catch (e, s) {
log(
"Error getting cover face for cluster $clusterID",
error: e,
stackTrace: s,
);
return;
}
}
@override
Widget build(BuildContext context) {
if (Platform.isIOS || Platform.isAndroid) {
if (useGeneratedFaceCrops) {
return FutureBuilder<Uint8List?>(
future: getFaceCrop(),
builder: (context, snapshot) {

7
mobile/lib/utils/face/face_box_crop.dart
View file

@ -5,13 +5,13 @@ import "package:photos/core/cache/lru_map.dart";
import "package:photos/face/model/box.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/file/file_type.dart";
import "package:photos/utils/face/face_util.dart";
import "package:photos/utils/file_util.dart";
import "package:photos/utils/image_ml_isolate.dart";
import "package:photos/utils/thumbnail_util.dart";
import "package:pool/pool.dart";
final LRUMap<String, Uint8List?> faceCropCache = LRUMap(1000);
final pool = Pool(5, timeout: const Duration(seconds: 15));
final pool = Pool(10, timeout: const Duration(seconds: 15));
Future<Map<String, Uint8List>?> getFaceCrops(
EnteFile file,
Map<String, FaceBox> faceBoxeMap,
@ -37,7 +37,8 @@ Future<Map<String, Uint8List>?> getFaceCrops(
faceBoxes.add(e.value);
}
final List<Uint8List> faceCrop =
await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
// await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
await generateJpgFaceThumbnails(
imagePath,
faceBoxes,
);

175
mobile/lib/utils/face/face_util.dart Normal file
View file

@ -0,0 +1,175 @@
import "dart:math";
import "dart:typed_data";
import "package:computer/computer.dart";
import "package:flutter_image_compress/flutter_image_compress.dart";
import "package:image/image.dart" as img;
import "package:logging/logging.dart";
import "package:photos/face/model/box.dart";
/// Bounding box of a face.
///
/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and
/// [width] and [height] are the width and height of the box.
///
/// One unit is equal to one pixel in the original image.
class FaceBoxImage {
final int xMin;
final int yMin;
final int width;
final int height;
FaceBoxImage({
required this.xMin,
required this.yMin,
required this.width,
required this.height,
});
}
final _logger = Logger("FaceUtil");
final _computer = Computer.shared();
const _faceImageBufferFactor = 0.2;
///Convert img.Image to ui.Image and use RawImage to display.
Future<List<img.Image>> generateImgFaceThumbnails(
String imagePath,
List<FaceBox> faceBoxes,
) async {
final faceThumbnails = <img.Image>[];
final image = await decodeToImgImage(imagePath);
for (FaceBox faceBox in faceBoxes) {
final croppedImage = cropFaceBoxFromImage(image, faceBox);
faceThumbnails.add(croppedImage);
}
return faceThumbnails;
}
Future<List<Uint8List>> generateJpgFaceThumbnails(
String imagePath,
List<FaceBox> faceBoxes,
) async {
final image = await decodeToImgImage(imagePath);
final croppedImages = <img.Image>[];
for (FaceBox faceBox in faceBoxes) {
final croppedImage = cropFaceBoxFromImage(image, faceBox);
croppedImages.add(croppedImage);
}
return await _computer
.compute(_encodeImagesToJpg, param: {"images": croppedImages});
}
Future<img.Image> decodeToImgImage(String imagePath) async {
img.Image? image =
await _computer.compute(_decodeImageFile, param: {"filePath": imagePath});
if (image == null) {
_logger.info(
"Failed to decode image. Compressing to jpg and decoding",
);
final compressedJPGImage =
await FlutterImageCompress.compressWithFile(imagePath);
image = await _computer.compute(
_decodeJpg,
param: {"image": compressedJPGImage},
);
if (image == null) {
throw Exception("Failed to decode image");
} else {
return image;
}
} else {
return image;
}
}
/// Returns an Image from 'package:image/image.dart'
img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) {
final squareFaceBox = _getSquareFaceBoxImage(image, faceBox);
final squareFaceBoxWithBuffer =
_addBufferAroundFaceBox(squareFaceBox, _faceImageBufferFactor);
return img.copyCrop(
image,
x: squareFaceBoxWithBuffer.xMin,
y: squareFaceBoxWithBuffer.yMin,
width: squareFaceBoxWithBuffer.width,
height: squareFaceBoxWithBuffer.height,
antialias: false,
);
}
/// Returns a square face box image from the original image with
/// side length equal to the maximum of the width and height of the face box in
/// the OG image.
FaceBoxImage _getSquareFaceBoxImage(img.Image image, FaceBox faceBox) {
final width = (image.width * faceBox.width).round();
final height = (image.height * faceBox.height).round();
final side = max(width, height);
final xImage = (image.width * faceBox.xMin).round();
final yImage = (image.height * faceBox.yMin).round();
if (height >= width) {
final xImageAdj = (xImage - (height - width) / 2).round();
return FaceBoxImage(
xMin: xImageAdj,
yMin: yImage,
width: side,
height: side,
);
} else {
final yImageAdj = (yImage - (width - height) / 2).round();
return FaceBoxImage(
xMin: xImage,
yMin: yImageAdj,
width: side,
height: side,
);
}
}
///To add some buffer around the face box so that the face isn't cropped
///too close to the face.
FaceBoxImage _addBufferAroundFaceBox(
FaceBoxImage faceBoxImage,
double bufferFactor,
) {
final heightBuffer = faceBoxImage.height * bufferFactor;
final widthBuffer = faceBoxImage.width * bufferFactor;
final xMinWithBuffer = faceBoxImage.xMin - widthBuffer;
final yMinWithBuffer = faceBoxImage.yMin - heightBuffer;
final widthWithBuffer = faceBoxImage.width + 2 * widthBuffer;
final heightWithBuffer = faceBoxImage.height + 2 * heightBuffer;
//Do not add buffer if the top left edge of the image is out of bounds
//after adding the buffer.
if (xMinWithBuffer < 0 || yMinWithBuffer < 0) {
return faceBoxImage;
}
//Another similar case that can be handled is when the bottom right edge
//of the image is out of bounds after adding the buffer. But the
//the visual difference is not as significant as when the top left edge
//is out of bounds, so we are not handling that case.
return FaceBoxImage(
xMin: xMinWithBuffer.round(),
yMin: yMinWithBuffer.round(),
width: widthWithBuffer.round(),
height: heightWithBuffer.round(),
);
}
List<Uint8List> _encodeImagesToJpg(Map args) {
final images = args["images"] as List<img.Image>;
return images.map((img.Image image) => img.encodeJpg(image)).toList();
}
Future<img.Image?> _decodeImageFile(Map args) async {
return await img.decodeImageFile(args["filePath"]);
}
img.Image? _decodeJpg(Map args) {
return img.decodeJpg(args["image"])!;
}

34
mobile/lib/utils/image_util.dart
View file

@ -1,6 +1,8 @@
import 'dart:async';
import 'dart:ui' as ui;
import 'package:flutter/widgets.dart';
import 'package:image/image.dart' as img;
Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
final completer = Completer<ImageInfo>();
@ -14,3 +16,35 @@ Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
completer.future.whenComplete(() => imageStream.removeListener(listener));
return completer.future;
}
Future<ui.Image> convertImageToFlutterUi(img.Image image) async {
if (image.format != img.Format.uint8 || image.numChannels != 4) {
final cmd = img.Command()
..image(image)
..convert(format: img.Format.uint8, numChannels: 4);
final rgba8 = await cmd.getImageThread();
if (rgba8 != null) {
image = rgba8;
}
}
final ui.ImmutableBuffer buffer =
await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
final ui.ImageDescriptor id = ui.ImageDescriptor.raw(
buffer,
height: image.height,
width: image.width,
pixelFormat: ui.PixelFormat.rgba8888,
);
final ui.Codec codec = await id.instantiateCodec(
targetHeight: image.height,
targetWidth: image.width,
);
final ui.FrameInfo fi = await codec.getNextFrame();
final ui.Image uiImage = fi.image;
return uiImage;
}