Merge branch 'main' into gallery_refactor_1

lib/db/device_files_db.dart

@@ -340,8 +340,9 @@ extension DeviceFiles on FilesDB {
   ) async {
     final db = await database;
     const String rawQuery = ''' 
-    SELECT ${FilesDB.columnLocalID}, ${FilesDB.columnUploadedFileID}
-          FROM ${FilesDB.filesTable}
+    SELECT ${FilesDB.columnLocalID}, ${FilesDB.columnUploadedFileID}, 
+    ${FilesDB.columnFileSize} 
+    FROM ${FilesDB.filesTable}
           WHERE ${FilesDB.columnLocalID} IS NOT NULL AND
           (${FilesDB.columnOwnerID} IS NULL OR ${FilesDB.columnOwnerID} = ?)
           AND (${FilesDB.columnUploadedFileID} IS NOT NULL AND ${FilesDB.columnUploadedFileID} IS NOT -1)
@@ -352,12 +353,17 @@ extension DeviceFiles on FilesDB {
     final results = await db.rawQuery(rawQuery, [ownerID, pathID]);
     final localIDs = <String>{};
     final uploadedIDs = <int>{};
+    int localSize = 0;
     for (final result in results) {
-      // FilesDB.[columnLocalID,columnUploadedFileID] is not null check in query
-      localIDs.add(result[FilesDB.columnLocalID] as String);
+      final String localID = result[FilesDB.columnLocalID] as String;
+      final int? fileSize = result[FilesDB.columnFileSize] as int?;
+      if (!localIDs.contains(localID) && fileSize != null) {
+        localSize += fileSize;
+      }
+      localIDs.add(localID);
       uploadedIDs.add(result[FilesDB.columnUploadedFileID] as int);
     }
-    return BackedUpFileIDs(localIDs.toList(), uploadedIDs.toList());
+    return BackedUpFileIDs(localIDs.toList(), uploadedIDs.toList(), localSize);
   }
 
   Future<List<DeviceCollection>> getDeviceCollections({

lib/db/files_db.dart

@@ -481,17 +481,23 @@ class FilesDB {
     final db = await instance.database;
     final results = await db.query(
       filesTable,
-      columns: [columnLocalID, columnUploadedFileID],
+      columns: [columnLocalID, columnUploadedFileID, columnFileSize],
       where:
           '$columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)',
     );
-    final localIDs = <String>{};
-    final uploadedIDs = <int>{};
+    final Set<String> localIDs = <String>{};
+    final Set<int> uploadedIDs = <int>{};
+    int localSize = 0;
     for (final result in results) {
+      final String localID = result[columnLocalID] as String;
+      final int? fileSize = result[columnFileSize] as int?;
+      if (!localIDs.contains(localID) && fileSize != null) {
+        localSize += fileSize;
+      }
       localIDs.add(result[columnLocalID] as String);
       uploadedIDs.add(result[columnUploadedFileID] as int);
     }
-    return BackedUpFileIDs(localIDs.toList(), uploadedIDs.toList());
+    return BackedUpFileIDs(localIDs.toList(), uploadedIDs.toList(), localSize);
   }
 
   Future<FileLoadResult> getAllPendingOrUploadedFiles(
@@ -1431,6 +1437,44 @@ class FilesDB {
     return _deduplicatedAndFilterIgnoredFiles(convertToFiles(rows), {});
   }
 
+  // For a given userID, return unique uploadedFileId for the given userID
+  Future<List<int>> getUploadIDsWithMissingSize(int userId) async {
+    final db = await instance.database;
+    final rows = await db.query(
+      filesTable,
+      columns: [columnUploadedFileID],
+      distinct: true,
+      where: '$columnOwnerID = ? AND $columnFileSize IS NULL',
+      whereArgs: [userId],
+    );
+    final result = <int>[];
+    for (final row in rows) {
+      result.add(row[columnUploadedFileID] as int);
+    }
+    return result;
+  }
+
+  // updateSizeForUploadIDs takes a map of upploadedFileID and fileSize and
+  // update the fileSize for the given uploadedFileID
+  Future<void> updateSizeForUploadIDs(
+    Map<int, int> uploadedFileIDToSize,
+  ) async {
+    if (uploadedFileIDToSize.isEmpty) {
+      return;
+    }
+    final db = await instance.database;
+    final batch = db.batch();
+    for (final uploadedFileID in uploadedFileIDToSize.keys) {
+      batch.update(
+        filesTable,
+        {columnFileSize: uploadedFileIDToSize[uploadedFileID]},
+        where: '$columnUploadedFileID = ?',
+        whereArgs: [uploadedFileID],
+      );
+    }
+    await batch.commit(noResult: true);
+  }
+
   Future<List<File>> getAllFilesFromDB(Set<int> collectionsToIgnore) async {
     final db = await instance.database;
     final List<Map<String, dynamic>> result =

lib/generated/intl/messages_nl.dart

@@ -381,6 +381,8 @@ class MessageLookup extends MessageLookupByLibrary {
         "close": MessageLookupByLibrary.simpleMessage("Sluiten"),
         "clubByCaptureTime":
             MessageLookupByLibrary.simpleMessage("Samenvoegen op tijd"),
+        "clubByFileName":
+            MessageLookupByLibrary.simpleMessage("Samenvoegen op bestandsnaam"),
         "codeAppliedPageTitle":
             MessageLookupByLibrary.simpleMessage("Code toegepast"),
         "codeCopiedToClipboard": MessageLookupByLibrary.simpleMessage(
@@ -396,6 +398,9 @@ class MessageLookup extends MessageLookupByLibrary {
         "collaboratorsCanAddPhotosAndVideosToTheSharedAlbum":
             MessageLookupByLibrary.simpleMessage(
                 "Samenwerkers kunnen foto\'s en video\'s toevoegen aan het gedeelde album."),
+        "collageLayout": MessageLookupByLibrary.simpleMessage("Layout"),
+        "collageSaved": MessageLookupByLibrary.simpleMessage(
+            "Collage opgeslagen in gallerij"),
         "collectEventPhotos": MessageLookupByLibrary.simpleMessage(
             "Foto\'s van gebeurtenissen verzamelen"),
         "collectPhotos":
@@ -440,6 +445,7 @@ class MessageLookup extends MessageLookupByLibrary {
             MessageLookupByLibrary.simpleMessage("Account aanmaken"),
         "createAlbumActionHint": MessageLookupByLibrary.simpleMessage(
             "Lang indrukken om foto\'s te selecteren en klik + om een album te maken"),
+        "createCollage": MessageLookupByLibrary.simpleMessage("Creëer collage"),
         "createNewAccount":
             MessageLookupByLibrary.simpleMessage("Nieuw account aanmaken"),
         "createOrSelectAlbum":
@@ -639,6 +645,8 @@ class MessageLookup extends MessageLookupByLibrary {
         "favorite":
             MessageLookupByLibrary.simpleMessage("Toevoegen aan favorieten"),
         "feedback": MessageLookupByLibrary.simpleMessage("Feedback"),
+        "fileFailedToSaveToGallery": MessageLookupByLibrary.simpleMessage(
+            "Opslaan van bestand naar galerij mislukt"),
         "fileInfoAddDescHint": MessageLookupByLibrary.simpleMessage(
             "Voeg een beschrijving toe..."),
         "fileSavedToGallery": MessageLookupByLibrary.simpleMessage(
@@ -688,6 +696,7 @@ class MessageLookup extends MessageLookupByLibrary {
         "iOSLockOut": MessageLookupByLibrary.simpleMessage(
             "Biometrische verificatie is uitgeschakeld. Vergrendel en ontgrendel uw scherm om het in te schakelen."),
         "iOSOkButton": MessageLookupByLibrary.simpleMessage("Oké"),
+        "ignoreUpdate": MessageLookupByLibrary.simpleMessage("Negeren"),
         "ignoredFolderUploadReason": MessageLookupByLibrary.simpleMessage(
             "Sommige bestanden in dit album worden genegeerd voor de upload omdat ze eerder van ente zijn verwijderd."),
         "importing": MessageLookupByLibrary.simpleMessage("Importeren...."),
@@ -1030,6 +1039,7 @@ class MessageLookup extends MessageLookupByLibrary {
         "safelyStored":
             MessageLookupByLibrary.simpleMessage("Veilig opgeslagen"),
         "save": MessageLookupByLibrary.simpleMessage("Opslaan"),
+        "saveCollage": MessageLookupByLibrary.simpleMessage("Sla collage op"),
         "saveCopy": MessageLookupByLibrary.simpleMessage("Kopie opslaan"),
         "saveKey": MessageLookupByLibrary.simpleMessage("Bewaar sleutel"),
         "saveYourRecoveryKeyIfYouHaventAlready":
@@ -1140,6 +1150,10 @@ class MessageLookup extends MessageLookupByLibrary {
         "sorryWeCouldNotGenerateSecureKeysOnThisDevicennplease":
             MessageLookupByLibrary.simpleMessage(
                 "Sorry, we konden geen beveiligde sleutels genereren op dit apparaat.\n\nGelieve je aan te melden vanaf een ander apparaat."),
+        "sortAlbumsBy": MessageLookupByLibrary.simpleMessage("Sorteren op"),
+        "sortNewestFirst":
+            MessageLookupByLibrary.simpleMessage("Nieuwste eerst"),
+        "sortOldestFirst": MessageLookupByLibrary.simpleMessage("Oudste eerst"),
         "sparkleSuccess": MessageLookupByLibrary.simpleMessage("✨ Succes"),
         "startBackup": MessageLookupByLibrary.simpleMessage("Back-up starten"),
         "storage": MessageLookupByLibrary.simpleMessage("Opslagruimte"),

lib/l10n/intl_it.arb

@@ -445,6 +445,7 @@
   "installManually": "Installa manualmente",
   "criticalUpdateAvailable": "Un aggiornamento importante è disponibile",
   "updateAvailable": "Aggiornamento disponibile",
+  "ignoreUpdate": "Ignora",
   "downloading": "Scaricamento in corso...",
   "theDownloadCouldNotBeCompleted": "Il download non può essere completato",
   "retry": "Riprova",
@@ -686,6 +687,10 @@
   "favorite": "Preferito",
   "removeFromFavorite": "Rimuovi dai preferiti",
   "shareLink": "Condividi link",
+  "createCollage": "Crea un collage",
+  "saveCollage": "Salva il collage",
+  "collageSaved": "Collage salvato nella galleria",
+  "collageLayout": "Disposizione",
   "addToEnte": "Aggiungi su ente",
   "addToAlbum": "Aggiungi all'album",
   "delete": "Cancella",
@@ -725,7 +730,7 @@
   "movedSuccessfullyTo": "Spostato con successo su {albumName}",
   "thisAlbumAlreadyHDACollaborativeLink": "Questo album ha già un link collaborativo",
   "collaborativeLinkCreatedFor": "Link collaborativo creato per {albumName}",
-  "askYourLovedOnesToShare": "Chiedi ai tuoi cari di condividere",
+  "askYourLovedOnesToShare": "Invita amici, amiche e parenti su ente",
   "invite": "Invita",
   "shareYourFirstAlbum": "Condividi il tuo primo album",
   "sharedWith": "Condiviso con {emailIDs}",
@@ -747,6 +752,9 @@
   },
   "deleteAll": "Elimina tutto",
   "renameAlbum": "Rinomina album",
+  "sortAlbumsBy": "Ordina per",
+  "sortNewestFirst": "Prima le più nuove",
+  "sortOldestFirst": "Prima le più vecchie",
   "rename": "Rinomina",
   "leaveSharedAlbum": "Abbandonare l'album condiviso?",
   "leaveAlbum": "Abbandona l'album",
@@ -783,6 +791,7 @@
   "close": "Chiudi",
   "setAs": "Imposta come",
   "fileSavedToGallery": "File salvato nella galleria",
+  "fileFailedToSaveToGallery": "Impossibile salvare il file nella galleria",
   "download": "Scarica",
   "pressAndHoldToPlayVideo": "Tieni premuto per riprodurre il video",
   "downloadFailed": "Scaricamento fallito",
@@ -790,6 +799,7 @@
   "deselectAll": "Deseleziona tutti",
   "reviewDeduplicateItems": "Controlla ed elimina gli elementi che credi siano dei doppioni.",
   "clubByCaptureTime": "Club per tempo di cattura",
+  "clubByFileName": "Unisci per nome file",
   "count": "Conteggio",
   "totalSize": "Dimensioni totali",
   "time": "Ora",

lib/models/backup_status.dart

@@ -8,6 +8,10 @@ class BackupStatus {
 class BackedUpFileIDs {
   final List<String> localIDs;
   final List<int> uploadedIDs;
+  // localSize indicates the approximate size of the files on the device.
+  // The size may not be exact because the remoteFile is encrypted before
+  // uploaded
+  final int localSize;
 
-  BackedUpFileIDs(this.localIDs, this.uploadedIDs);
+  BackedUpFileIDs(this.localIDs, this.uploadedIDs, this.localSize);
 }

lib/services/deduplication_service.dart

@@ -1,9 +1,12 @@
 import 'package:logging/logging.dart';
+import "package:photos/core/configuration.dart";
 import 'package:photos/core/errors.dart';
 import 'package:photos/core/network/network.dart';
 import 'package:photos/db/files_db.dart';
 import 'package:photos/models/duplicate_files.dart';
 import 'package:photos/models/file.dart';
+import "package:photos/services/collections_service.dart";
+import "package:photos/services/files_service.dart";
 
 class DeduplicationService {
   final _logger = Logger("DeduplicationService");
@@ -16,6 +19,11 @@ class DeduplicationService {
 
   Future<List<DuplicateFiles>> getDuplicateFiles() async {
     try {
+      final bool hasFileSizes = await FilesService.instance.hasMigratedSizes();
+      if (hasFileSizes) {
+        final List<DuplicateFiles> result = await _getDuplicateFilesFromLocal();
+        return result;
+      }
       final DuplicateFilesResponse dupes = await _fetchDuplicateFileIDs();
       final ids = <int>[];
       for (final dupe in dupes.duplicates) {
@@ -60,8 +68,8 @@ class DeduplicationService {
         );
       }
       return result;
-    } catch (e) {
-      _logger.severe(e);
+    } catch (e, s) {
+      _logger.severe("failed to get dedupeFile", e, s);
       rethrow;
     }
   }
@@ -95,6 +103,33 @@ class DeduplicationService {
     return dupesBySizeAndClubKey;
   }
 
+  Future<List<DuplicateFiles>> _getDuplicateFilesFromLocal() async {
+    final List<File> allFiles = await FilesDB.instance
+        .getAllFilesFromDB(CollectionsService.instance.getHiddenCollections());
+    final int ownerID = Configuration.instance.getUserID()!;
+    allFiles.removeWhere(
+      (f) =>
+          !f.isUploaded ||
+          (f.ownerID ?? 0) != ownerID ||
+          (f.fileSize ?? 0) <= 0,
+    );
+    final Map<int, List<File>> sizeToFilesMap = {};
+    for (final file in allFiles) {
+      if (!sizeToFilesMap.containsKey(file.fileSize)) {
+        sizeToFilesMap[file.fileSize!] = <File>[];
+      }
+      sizeToFilesMap[file.fileSize]!.add(file);
+    }
+    final List<DuplicateFiles> dupesBySize = [];
+    for (final size in sizeToFilesMap.keys) {
+      final List<File> files = sizeToFilesMap[size]!;
+      if (files.length > 1) {
+        dupesBySize.add(DuplicateFiles(files, size));
+      }
+    }
+    return dupesBySize;
+  }
+
   Future<DuplicateFilesResponse> _fetchDuplicateFileIDs() async {
     final response = await _enteDio.get("/files/duplicates");
     return DuplicateFilesResponse.fromMap(response.data);

lib/services/files_service.dart

@@ -42,6 +42,45 @@ class FilesService {
     }
   }
 
+  Future<bool> hasMigratedSizes() async {
+    try {
+      final List<int> uploadIDsWithMissingSize =
+          await _filesDB.getUploadIDsWithMissingSize(_config.getUserID()!);
+      if (uploadIDsWithMissingSize.isEmpty) {
+        return Future.value(true);
+      }
+      final batchedFiles = uploadIDsWithMissingSize.chunks(1000);
+      for (final batch in batchedFiles) {
+        final Map<int, int> uploadIdToSize = await getFilesSizeFromInfo(batch);
+        await _filesDB.updateSizeForUploadIDs(uploadIdToSize);
+      }
+      return Future.value(true);
+    } catch (e, s) {
+      _logger.severe("error during has migrated sizes", e, s);
+      return Future.value(false);
+    }
+  }
+
+  Future<Map<int, int>> getFilesSizeFromInfo(List<int> uploadedFileID) async {
+    try {
+      final response = await _enteDio.post(
+        "/files/info",
+        data: {"fileIDs": uploadedFileID},
+      );
+      final Map<int, int> idToSize = {};
+      final List result = response.data["filesInfo"] as List;
+      for (var fileInfo in result) {
+        final int uploadedFileID = fileInfo["id"];
+        final int size = fileInfo["fileInfo"]["fileSize"];
+        idToSize[uploadedFileID] = size;
+      }
+      return idToSize;
+    } catch (e, s) {
+      _logger.severe("failed to fetch size from fileInfo", e, s);
+      rethrow;
+    }
+  }
+
   // Note: this method is not used anywhere, but it is kept for future
   // reference when we add bulk EditTime feature
   Future<void> bulkEditTime(

lib/services/sync_service.dart

@@ -18,6 +18,7 @@ import 'package:photos/events/sync_status_update_event.dart';
 import 'package:photos/events/trigger_logout_event.dart';
 import 'package:photos/models/backup_status.dart';
 import 'package:photos/models/file_type.dart';
+import "package:photos/services/files_service.dart";
 import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/services/notification_service.dart';
 import 'package:photos/services/remote_sync_service.dart';
@@ -209,6 +210,7 @@ class SyncService {
 
   Future<BackupStatus> getBackupStatus({String? pathID}) async {
     BackedUpFileIDs ids;
+    final bool hasMigratedSize = await FilesService.instance.hasMigratedSizes();
     if (pathID == null) {
       ids = await FilesDB.instance.getBackedUpIDs();
     } else {
@@ -217,7 +219,12 @@ class SyncService {
         Configuration.instance.getUserID()!,
       );
     }
-    final size = await _getFileSize(ids.uploadedIDs);
+    late int size;
+    if (hasMigratedSize) {
+      size = ids.localSize;
+    } else {
+      size = await _getFileSize(ids.uploadedIDs);
+    }
     return BackupStatus(ids.localIDs, size);
   }
 

pubspec.yaml

@@ -12,7 +12,7 @@ description: ente photos application
 # Read more about iOS versioning at
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
 
-version: 0.7.60+460
+version: 0.7.61+461
 
 environment:
   sdk: '>=2.17.0 <3.0.0'