0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[mob][photos] Dedupe using hash before copying

Browse source
This commit is contained in:
Neeraj Gupta 2024-04-19 16:45:29 +05:30
parent 02890a100b
commit 01822b760d
2 changed files with 59 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
mobile/lib/db/files_db.dart
View file

@ -521,7 +521,8 @@ class FilesDB {
}
Future<(Set<int>, Map<String, int>)> getUploadAndHash(
int collectionID,) async {
int collectionID,
) async {
final db = await instance.database;
final results = await db.query(
filesTable,
@ -1045,6 +1046,29 @@ class FilesDB {
return convertToFiles(rows);
}
Future<Map<String, EnteFile>>
getUserOwnedFilesWithSameHashForGivenListOfFiles(
List<EnteFile> files,
int userID,
) async {
final db = await sqliteAsyncDB;
final List<String> hashes = [];
for (final file in files) {
if (file.hash != null && file.hash != '') {
hashes.add(file.hash!);
}
}
if (hashes.isEmpty) {
return {};
}
final inParam = hashes.map((e) => "'$e'").join(',');
final rows = await db.execute('''
SELECT * FROM $filesTable WHERE $columnHash IN ($inParam) AND $columnOwnerID = $userID;
''');
final matchedFiles = convertToFiles(rows);
return Map.fromIterable(matchedFiles, key: (e) => e.hash);
}
Future<List<EnteFile>> getUploadedFilesWithHashes(
FileHashData hashData,
FileType fileType,

35
mobile/lib/services/collections_service.dart
View file

@ -1161,9 +1161,21 @@ class CollectionsService {
await _addToCollection(dstCollectionID, splitResult.ownedByCurrentUser);
}
if (splitResult.ownedByOtherUsers.isNotEmpty) {
late final List<EnteFile> filesToCopy;
late final List<EnteFile> filesToAdd;
(filesToAdd, filesToCopy) = (await _splitFilesToAddAndCopy(
splitResult.ownedByOtherUsers,
));
if (filesToAdd.isNotEmpty) {
_logger.info(
"found existing ${filesToAdd.length} files with same hash, adding symlinks",
);
await _addToCollection(dstCollectionID, filesToAdd);
}
// group files by collectionID
final Map<int, List<EnteFile>> filesByCollection = {};
for (final file in splitResult.ownedByOtherUsers) {
for (final file in filesToCopy) {
if (filesByCollection.containsKey(file.collectionID!)) {
filesByCollection[file.collectionID!]!.add(file);
} else {
@ -1365,6 +1377,27 @@ class CollectionsService {
}
}
Future<(List<EnteFile>, List<EnteFile>)> _splitFilesToAddAndCopy(
List<EnteFile> othersFile,
) async {
final hashToUserFile =
await _filesDB.getUserOwnedFilesWithSameHashForGivenListOfFiles(
othersFile,
_config.getUserID()!,
);
final List<EnteFile> filesToCopy = [];
final List<EnteFile> filesToAdd = [];
for (final EnteFile file in othersFile) {
if (hashToUserFile.containsKey(file.hash ?? '')) {
final userFile = hashToUserFile[file.hash]!;
filesToAdd.add(userFile);
} else {
filesToCopy.add(file);
}
}
return (filesToAdd, filesToCopy);
}
void _validateCopyInput(
int destCollectionID,
int srcCollectionID,