Merge branch 'rewrite_device_sync' into rewrite_device_sync_remote

lib/db/device_files_db.dart

@@ -48,7 +48,8 @@ extension DeviceFiles on FilesDB {
   }
 
   Future<void> deletePathIDToLocalIDMapping(
-      Map<String, Set<String>> mappingsToRemove) async {
+    Map<String, Set<String>> mappingsToRemove,
+  ) async {
     debugPrint("removing PathIDToLocalIDMapping");
     final db = await database;
     var batch = db.batch();

lib/db/file_migration_db.dart → lib/db/file_updation_db.dart

@@ -7,13 +7,13 @@ import 'package:path_provider/path_provider.dart';
 import 'package:sqflite/sqflite.dart';
 import 'package:sqflite_migration/sqflite_migration.dart';
 
-class FilesMigrationDB {
+class FileUpdationDB {
   static const _databaseName = "ente.files_migration.db";
-  static final Logger _logger = Logger((FilesMigrationDB).toString());
+  static final Logger _logger = Logger((FileUpdationDB).toString());
 
   static const tableName = 're_upload_tracker';
-  static const _columnLocalID = 'local_id';
-  static const _columnReason = 'reason';
+  static const columnLocalID = 'local_id';
+  static const columnReason = 'reason';
   static const missingLocation = 'missing_location';
   static const modificationTimeUpdated = 'modificationTimeUpdated';
 
@@ -22,8 +22,8 @@ class FilesMigrationDB {
     return [
       ''' 
       CREATE TABLE $tableName (
-      $_columnLocalID TEXT NOT NULL,
-      UNIQUE($_columnLocalID)
+      $columnLocalID TEXT NOT NULL,
+      UNIQUE($columnLocalID)
       ); 
       ''',
     ];
@@ -32,10 +32,10 @@ class FilesMigrationDB {
   static List<String> addReasonColumn() {
     return [
       '''
-        ALTER TABLE $tableName ADD COLUMN $_columnReason TEXT;
+        ALTER TABLE $tableName ADD COLUMN $columnReason TEXT;
       ''',
       '''
-        UPDATE $tableName SET $_columnReason = '$missingLocation';
+        UPDATE $tableName SET $columnReason = '$missingLocation';
       ''',
     ];
   }
@@ -49,10 +49,9 @@ class FilesMigrationDB {
     migrationScripts: migrationScripts,
   );
 
-  FilesMigrationDB._privateConstructor();
+  FileUpdationDB._privateConstructor();
 
-  static final FilesMigrationDB instance =
-      FilesMigrationDB._privateConstructor();
+  static final FileUpdationDB instance = FileUpdationDB._privateConstructor();
 
   // only have a single app-wide reference to the database
   static Future<Database> _dbFuture;
@@ -123,7 +122,7 @@ class FilesMigrationDB {
     await db.rawQuery(
       '''
       DELETE FROM $tableName
-      WHERE $_columnLocalID IN ($inParam) AND $_columnReason = '$reason';
+      WHERE $columnLocalID IN ($inParam) AND $columnReason = '$reason';
     ''',
     );
   }
@@ -133,7 +132,7 @@ class FilesMigrationDB {
     String reason,
   ) async {
     final db = await instance.database;
-    final String whereClause = '$_columnReason = "$reason"';
+    final String whereClause = '$columnReason = "$reason"';
     final rows = await db.query(
       tableName,
       limit: limit,
@@ -141,7 +140,7 @@ class FilesMigrationDB {
     );
     final result = <String>[];
     for (final row in rows) {
-      result.add(row[_columnLocalID]);
+      result.add(row[columnLocalID]);
     }
     return result;
   }
@@ -149,8 +148,8 @@ class FilesMigrationDB {
   Map<String, dynamic> _getRowForReUploadTable(String localID, String reason) {
     assert(localID != null);
     final row = <String, dynamic>{};
-    row[_columnLocalID] = localID;
-    row[_columnReason] = reason;
+    row[columnLocalID] = localID;
+    row[columnReason] = reason;
     return row;
   }
 }

lib/db/files_db.dart

@@ -11,6 +11,7 @@ import 'package:photos/models/file_load_result.dart';
 import 'package:photos/models/file_type.dart';
 import 'package:photos/models/location.dart';
 import 'package:photos/models/magic_metadata.dart';
+import 'package:photos/utils/file_uploader_util.dart';
 import 'package:sqflite/sqflite.dart';
 import 'package:sqflite_migration/sqflite_migration.dart';
 
@@ -82,6 +83,7 @@ class FilesDB {
     initializationScript: initializationScript,
     migrationScripts: migrationScripts,
   );
+
   // make this a singleton class
   FilesDB._privateConstructor();
 
@@ -498,7 +500,6 @@ class FilesDB {
           '$columnCreationTime ' + order + ', $columnModificationTime ' + order,
       limit: limit,
     );
-
     final files = convertToFiles(results);
     final List<File> deduplicatedFiles =
         _deduplicatedAndFilterIgnoredFiles(files, ignoredCollectionIDs);
@@ -883,23 +884,27 @@ class FilesDB {
   }
 
   Future<List<File>> getUploadedFilesWithHashes(
-    List<String> hash,
+    FileHashData hashData,
     FileType fileType,
     int ownerID,
   ) async {
-    // look up two hash at max, for handling live photos
-    assert(hash.length < 3, "number of hash can not be more than 2");
-    final db = await instance.database;
-    final String rawQuery =
-        'SELECT * from files where ($columnUploadedFileID != '
-        'NULL OR $columnUploadedFileID != -1) AND $columnOwnerID = $ownerID '
-        'AND ($columnHash = "${hash.first}" OR $columnHash = "${hash.last}")';
-    final rows = await db.rawQuery(rawQuery, []);
-    if (rows.isNotEmpty) {
-      return convertToFiles(rows);
-    } else {
-      return [];
+    String inParam = "'${hashData.fileHash}'";
+    if (fileType == FileType.livePhoto && hashData.zipHash != null) {
+      inParam += ",'${hashData.zipHash}'";
     }
+    final db = await instance.database;
+    final rows = await db.query(
+      filesTable,
+      where: '($columnUploadedFileID != NULL OR $columnUploadedFileID != -1) '
+          'AND $columnOwnerID = ? AND $columnFileType ='
+          ' ? '
+          'AND $columnHash IN ($inParam)',
+      whereArgs: [
+        ownerID,
+        getInt(fileType),
+      ],
+    );
+    return convertToFiles(rows);
   }
 
   Future<int> update(File file) async {

lib/db/ignored_files_db.dart

@@ -52,7 +52,8 @@ class IgnoredFilesDB {
 
   // this opens the database (and creates it if it doesn't exist)
   Future<Database> _initDatabase() async {
-    final Directory documentsDirectory = await getApplicationDocumentsDirectory();
+    final Directory documentsDirectory =
+        await getApplicationDocumentsDirectory();
     final String path = join(documentsDirectory.path, _databaseName);
     return await openDatabase(
       path,
@@ -117,10 +118,17 @@ class IgnoredFilesDB {
         batch = db.batch();
         batchCounter = 0;
       }
-      batch.rawDelete(
-        "DELETE from $tableName WHERE "
-        "$columnLocalID = '${file.localID}' OR ( $columnDeviceFolder = '${file.deviceFolder}' AND $columnTitle = '${file.title}' ) ",
-      );
+      // on Android, we track device folder and title to track files to ignore.
+      // See IgnoredFileService#_getIgnoreID method for more detail
+      if (Platform.isAndroid) {
+        batch.rawDelete(
+          "DELETE from $tableName WHERE  $columnDeviceFolder = '${file.deviceFolder}' AND $columnTitle = '${file.title}' ",
+        );
+      } else {
+        batch.rawDelete(
+          "DELETE from $tableName WHERE $columnLocalID = '${file.localID}' ",
+        );
+      }
       batchCounter++;
     }
     await batch.commit(noResult: true);

lib/models/file.dart

@@ -16,7 +16,6 @@ class File extends EnteFile {
   int ownerID;
   int collectionID;
   String localID;
-
   String title;
   String deviceFolder;
   int creationTime;
@@ -54,7 +53,9 @@ class File extends EnteFile {
 
   set pubMagicMetadata(val) => _pubMmd = val;
 
-  static const kCurrentMetadataVersion = 1;
+  // in Version 1, live photo hash is stored as zip's hash.
+  // in V2: LivePhoto hash is stored as imgHash:vidHash
+  static const kCurrentMetadataVersion = 2;
 
   File();
 
@@ -134,6 +135,15 @@ class File extends EnteFile {
     duration = metadata["duration"] ?? 0;
     exif = metadata["exif"];
     hash = metadata["hash"];
+    // handle past live photos upload from web client
+    if (hash == null &&
+        fileType == FileType.livePhoto &&
+        metadata.containsKey('imgHash') &&
+        metadata.containsKey('vidHash')) {
+      // convert to imgHash:vidHash
+      hash =
+          '${metadata['imgHash']}$kLivePhotoHashSeparator${metadata['vidHash']}';
+    }
     metadataVersion = metadata["version"] ?? 0;
   }
 
@@ -155,13 +165,7 @@ class File extends EnteFile {
         creationTime = exifTime.microsecondsSinceEpoch;
       }
     }
-    // in metadataVersion V1, the hash for livePhoto is the hash of the
-    // zipped file.
-    // web uploads files without MetadataVersion and upload image hash as 'ha
-    // sh' key and video as 'vidHash'
-    hash = (fileType == FileType.livePhoto)
-        ? mediaUploadData.zipHash
-        : mediaUploadData.fileHash;
+    hash = mediaUploadData.hashData?.fileHash;
     return getMetadata();
   }
 

lib/models/ignored_file.dart

@@ -1,4 +1,4 @@
-import 'package:photos/models/file.dart';
+import 'package:photos/models/trash_file.dart';
 
 const kIgnoreReasonTrash = "trash";
 const kIgnoreReasonInvalidFile = "invalidFile";
@@ -11,7 +11,7 @@ class IgnoredFile {
 
   IgnoredFile(this.localID, this.title, this.deviceFolder, this.reason);
 
-  factory IgnoredFile.fromFile(File trashFile) {
+  factory IgnoredFile.fromTrashItem(TrashFile trashFile) {
     if (trashFile == null) return null;
     if (trashFile.localID == null ||
         trashFile.localID.isEmpty ||

lib/services/collections_service.dart

@@ -631,29 +631,30 @@ class CollectionsService {
   }
 
   Future<void> linkLocalFileToExistingUploadedFileInAnotherCollection(
-    int destCollectionID,
-    File localFileToUpload,
-    File file,
-  ) async {
+    int destCollectionID, {
+    @required File localFileToUpload,
+    @required File existingUploadedFile,
+  }) async {
     final params = <String, dynamic>{};
     params["collectionID"] = destCollectionID;
     params["files"] = [];
+    final int uploadedFileID = existingUploadedFile.uploadedFileID;
 
-    final key = decryptFileKey(file);
-    file.generatedID = localFileToUpload.generatedID; // So that a new entry is
-    // created in the FilesDB
-    file.localID = localFileToUpload.localID;
-    file.collectionID = destCollectionID;
+    // encrypt the fileKey with destination collection's key
+    final fileKey = decryptFileKey(existingUploadedFile);
     final encryptedKeyData =
-        CryptoUtil.encryptSync(key, getCollectionKey(destCollectionID));
-    file.encryptedKey = Sodium.bin2base64(encryptedKeyData.encryptedData);
-    file.keyDecryptionNonce = Sodium.bin2base64(encryptedKeyData.nonce);
+        CryptoUtil.encryptSync(fileKey, getCollectionKey(destCollectionID));
+
+    localFileToUpload.encryptedKey =
+        Sodium.bin2base64(encryptedKeyData.encryptedData);
+    localFileToUpload.keyDecryptionNonce =
+        Sodium.bin2base64(encryptedKeyData.nonce);
 
     params["files"].add(
       CollectionFileItem(
-        file.uploadedFileID,
-        file.encryptedKey,
-        file.keyDecryptionNonce,
+        uploadedFileID,
+        localFileToUpload.encryptedKey,
+        localFileToUpload.keyDecryptionNonce,
       ).toMap(),
     );
 
@@ -665,8 +666,12 @@ class CollectionsService {
           headers: {"X-Auth-Token": Configuration.instance.getToken()},
         ),
       );
-      await _filesDB.insertMultiple([file]);
-      Bus.instance.fire(CollectionUpdatedEvent(destCollectionID, [file]));
+      localFileToUpload.collectionID = destCollectionID;
+      localFileToUpload.uploadedFileID = uploadedFileID;
+      await _filesDB.insertMultiple([localFileToUpload]);
+      Bus.instance.fire(
+        CollectionUpdatedEvent(destCollectionID, [localFileToUpload]),
+      );
     } catch (e) {
       rethrow;
     }

lib/services/ignored_files_service.dart

@@ -47,35 +47,29 @@ class IgnoredFilesService {
     return false;
   }
 
+  // removeIgnoredMappings is used to remove the ignore mapping for the given
+  // set of files so that they can be uploaded.
   Future<void> removeIgnoredMappings(List<File> files) async {
     final List<IgnoredFile> ignoredFiles = [];
     final Set<String> idsToRemoveFromCache = {};
-    for (var file in files) {
-      if (Platform.isIOS && file.localID != null) {
-        // in IOS, the imported file might not have title fetched by default.
-        // fetching title has performance impact.
-        if (file.title == null || file.title.isEmpty) {
-          file.title = 'dummyTitle';
-        }
-      }
-      final ignoredFile = IgnoredFile.fromFile(file);
-      if (ignoredFile != null) {
-        ignoredFiles.add(ignoredFile);
-        final id = _idForIgnoredFile(ignoredFile);
-        if (id != null) {
-          idsToRemoveFromCache.add(id);
-        }
-      } else {
-        _logger.warning(
-            'ignoredFile should not be null while removing mapping ${file.tag()}');
+    final Set<String> currentlyIgnoredIDs = await ignoredIDs;
+    for (final file in files) {
+      // check if upload is not skipped for file. If not, no need to remove
+      // any mapping
+      if (!shouldSkipUpload(currentlyIgnoredIDs, file)) {
+        continue;
       }
+      final id = _getIgnoreID(file.localID, file.deviceFolder, file.title);
+      idsToRemoveFromCache.add(id);
+      ignoredFiles.add(
+        IgnoredFile(file.localID, file.title, file.deviceFolder, ""),
+      );
     }
+
     if (ignoredFiles.isNotEmpty) {
       await _db.removeIgnoredEntries(ignoredFiles);
-      final existingIDs = await ignoredIDs;
-      existingIDs.removeAll(idsToRemoveFromCache);
+      currentlyIgnoredIDs.removeAll(idsToRemoveFromCache);
     }
-    return;
   }
 
   Future<Set<String>> _loadExistingIDs() async {
@@ -95,7 +89,7 @@ class IgnoredFilesService {
     );
   }
 
-  // _computeIgnoreID will return null if don't have sufficient information
+  // _getIgnoreID will return null if don't have sufficient information
   // to ignore the file based on the platform. Uploads from web or files shared to
   // end usually don't have local id.
   // For Android: It returns deviceFolder-title as ID for Android.

lib/services/local_file_update_service.dart

@@ -5,8 +5,9 @@ import 'dart:io';
 import 'package:flutter/foundation.dart';
 import 'package:logging/logging.dart';
 import 'package:photo_manager/photo_manager.dart';
-import 'package:photos/db/file_migration_db.dart';
+import 'package:photos/db/file_updation_db.dart';
 import 'package:photos/db/files_db.dart';
+import 'package:photos/models/file.dart' as ente;
 import 'package:photos/utils/file_uploader_util.dart';
 import 'package:shared_preferences/shared_preferences.dart';
 
@@ -14,7 +15,7 @@ import 'package:shared_preferences/shared_preferences.dart';
 // changed/modified on the device and needed to be uploaded again.
 class LocalFileUpdateService {
   FilesDB _filesDB;
-  FilesMigrationDB _filesMigrationDB;
+  FileUpdationDB _fileUpdationDB;
   SharedPreferences _prefs;
   Logger _logger;
   static const isLocationMigrationComplete = "fm_isLocationMigrationComplete";
@@ -24,7 +25,7 @@ class LocalFileUpdateService {
   LocalFileUpdateService._privateConstructor() {
     _logger = Logger((LocalFileUpdateService).toString());
     _filesDB = FilesDB.instance;
-    _filesMigrationDB = FilesMigrationDB.instance;
+    _fileUpdationDB = FileUpdationDB.instance;
   }
 
   Future<void> init() async {
@@ -55,11 +56,10 @@ class LocalFileUpdateService {
         await _runMigrationForFilesWithMissingLocation();
       }
       await _markFilesWhichAreActuallyUpdated();
-      _existingMigration.complete();
-      _existingMigration = null;
     } catch (e, s) {
       _logger.severe('failed to perform migration', e, s);
-      _existingMigration.complete();
+    } finally {
+      _existingMigration?.complete();
       _existingMigration = null;
     }
   }
@@ -74,9 +74,9 @@ class LocalFileUpdateService {
     const int limitInBatch = 100;
     while (hasData) {
       final localIDsToProcess =
-          await _filesMigrationDB.getLocalIDsForPotentialReUpload(
+          await _fileUpdationDB.getLocalIDsForPotentialReUpload(
         limitInBatch,
-        FilesMigrationDB.modificationTimeUpdated,
+        FileUpdationDB.modificationTimeUpdated,
       );
       if (localIDsToProcess.isEmpty) {
         hasData = false;
@@ -97,18 +97,21 @@ class LocalFileUpdateService {
     List<String> localIDsToProcess,
   ) async {
     _logger.info("files to process ${localIDsToProcess.length} for reupload");
-    final localFiles = await FilesDB.instance.getLocalFiles(localIDsToProcess);
+    final List<ente.File> localFiles =
+        await FilesDB.instance.getLocalFiles(localIDsToProcess);
     final Set<String> processedIDs = {};
-    for (var file in localFiles) {
+    for (ente.File file in localFiles) {
       if (processedIDs.contains(file.localID)) {
         continue;
       }
       MediaUploadData uploadData;
       try {
-        uploadData = await getUploadDataFromEnteFile(file);
-        if (file.hash != null ||
-            (file.hash == uploadData.fileHash ||
-                file.hash == uploadData.zipHash)) {
+        uploadData = await getUploadData(file);
+        if (uploadData != null &&
+            uploadData.hashData != null &&
+            file.hash != null &&
+            (file.hash == uploadData.hashData.fileHash ||
+                file.hash == uploadData.hashData.zipHash)) {
           _logger.info("Skip file update as hash matched ${file.tag()}");
         } else {
           _logger.info(
@@ -126,24 +129,28 @@ class LocalFileUpdateService {
         processedIDs.add(file.localID);
       } catch (e) {
         _logger.severe("Failed to get file uploadData", e);
-      } finally {
-        // delete the file from app's internal cache if it was copied to app
-        // for upload. Shared Media should only be cleared when the upload
-        // succeeds.
-        if (Platform.isIOS &&
-            uploadData != null &&
-            uploadData.sourceFile != null) {
-          await uploadData.sourceFile.delete();
-        }
-      }
+      } finally {}
     }
     debugPrint("Deleting files ${processedIDs.length}");
-    await _filesMigrationDB.deleteByLocalIDs(
+    await _fileUpdationDB.deleteByLocalIDs(
       processedIDs.toList(),
-      FilesMigrationDB.modificationTimeUpdated,
+      FileUpdationDB.modificationTimeUpdated,
     );
   }
 
+  Future<MediaUploadData> getUploadData(ente.File file) async {
+    final mediaUploadData = await getUploadDataFromEnteFile(file);
+    // delete the file from app's internal cache if it was copied to app
+    // for upload. Shared Media should only be cleared when the upload
+    // succeeds.
+    if (Platform.isIOS &&
+        mediaUploadData != null &&
+        mediaUploadData.sourceFile != null) {
+      await mediaUploadData.sourceFile.delete();
+    }
+    return mediaUploadData;
+  }
+
   Future<void> _runMigrationForFilesWithMissingLocation() async {
     if (!Platform.isAndroid) {
       return;
@@ -158,9 +165,9 @@ class LocalFileUpdateService {
       const int limitInBatch = 100;
       while (hasData) {
         final localIDsToProcess =
-            await _filesMigrationDB.getLocalIDsForPotentialReUpload(
+            await _fileUpdationDB.getLocalIDsForPotentialReUpload(
           limitInBatch,
-          FilesMigrationDB.missingLocation,
+          FileUpdationDB.missingLocation,
         );
         if (localIDsToProcess.isEmpty) {
           hasData = false;
@@ -206,9 +213,9 @@ class LocalFileUpdateService {
     }
     _logger.info('marking ${localIDsWithLocation.length} files for re-upload');
     await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation);
-    await _filesMigrationDB.deleteByLocalIDs(
+    await _fileUpdationDB.deleteByLocalIDs(
       localIDsToProcess,
-      FilesMigrationDB.missingLocation,
+      FileUpdationDB.missingLocation,
     );
   }
 
@@ -219,9 +226,9 @@ class LocalFileUpdateService {
     final sTime = DateTime.now().microsecondsSinceEpoch;
     _logger.info('importing files without location info');
     final fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation();
-    await _filesMigrationDB.insertMultiple(
+    await _fileUpdationDB.insertMultiple(
       fileLocalIDs,
-      FilesMigrationDB.missingLocation,
+      FileUpdationDB.missingLocation,
     );
     final eTime = DateTime.now().microsecondsSinceEpoch;
     final d = Duration(microseconds: eTime - sTime);

lib/services/local_sync_service.dart

@@ -9,7 +9,7 @@ import 'package:photo_manager/photo_manager.dart';
 import 'package:photos/core/configuration.dart';
 import 'package:photos/core/event_bus.dart';
 import 'package:photos/db/device_files_db.dart';
-import 'package:photos/db/file_migration_db.dart';
+import 'package:photos/db/file_updation_db.dart';
 import 'package:photos/db/files_db.dart';
 import 'package:photos/db/ignored_files_db.dart';
 import 'package:photos/events/local_photos_updated_event.dart';
@@ -121,7 +121,7 @@ class LocalSyncService {
     if (!_prefs.containsKey(kHasCompletedFirstImportKey) ||
         !_prefs.getBool(kHasCompletedFirstImportKey)) {
       await _prefs.setBool(kHasCompletedFirstImportKey, true);
-      await refreshDeviceFolderCountAndCover();
+      await _refreshDeviceFolderCountAndCover();
       _logger.fine("first gallery import finished");
       Bus.instance
           .fire(SyncStatusUpdate(SyncStatus.completedFirstGalleryImport));
@@ -133,7 +133,7 @@ class LocalSyncService {
     _existingSync = null;
   }
 
-  Future<bool> refreshDeviceFolderCountAndCover() async {
+  Future<bool> _refreshDeviceFolderCountAndCover() async {
     final List<Tuple2<AssetPathEntity, String>> result =
         await getDeviceFolderWithCountAndCoverID();
     return await _db.updateDeviceCoverWithCount(
@@ -148,7 +148,7 @@ class LocalSyncService {
     _logger.info(
       "Loading allLocalAssets ${localAssets.length} took ${stopwatch.elapsedMilliseconds}ms ",
     );
-    await refreshDeviceFolderCountAndCover();
+    await _refreshDeviceFolderCountAndCover();
     _logger.info(
       "refreshDeviceFolderCountAndCover + allLocalAssets took ${stopwatch.elapsedMilliseconds}ms ",
     );
@@ -337,13 +337,11 @@ class LocalSyncService {
       );
       final List<String> updatedLocalIDs = [];
       for (final file in updatedFiles) {
-        if (file.localID != null) {
-          updatedLocalIDs.add(file.localID);
-        }
+        updatedLocalIDs.add(file.localID);
       }
-      await FilesMigrationDB.instance.insertMultiple(
+      await FileUpdationDB.instance.insertMultiple(
         updatedLocalIDs,
-        FilesMigrationDB.modificationTimeUpdated,
+        FileUpdationDB.modificationTimeUpdated,
       );
     }
   }

lib/services/trash_sync_service.dart

@@ -67,7 +67,7 @@ class TrashSyncService {
   Future<void> _updateIgnoredFiles(Diff diff) async {
     final ignoredFiles = <IgnoredFile>[];
     for (TrashFile t in diff.trashedFiles) {
-      final file = IgnoredFile.fromFile(t);
+      final file = IgnoredFile.fromTrashItem(t);
       if (file != null) {
         ignoredFiles.add(file);
       }

lib/ui/create_collection_page.dart

@@ -285,7 +285,8 @@ class _CreateCollectionPageState extends State<CreateCollectionPage> {
     final dialog = createProgressDialog(context, "Moving files to album...");
     await dialog.show();
     try {
-      final int fromCollectionID = widget.selectedFiles.files?.first?.collectionID;
+      final int fromCollectionID =
+          widget.selectedFiles.files?.first?.collectionID;
       await CollectionsService.instance.move(
         toCollectionID,
         fromCollectionID,
@@ -355,7 +356,10 @@ class _CreateCollectionPageState extends State<CreateCollectionPage> {
         }
       }
       if (filesPendingUpload.isNotEmpty) {
-        await IgnoredFilesService.instance.removeIgnoredMappings(filesPendingUpload);
+        // filesPendingUpload might be getting ignored during auto-upload
+        // because the user deleted these files from ente in the past.
+        await IgnoredFilesService.instance
+            .removeIgnoredMappings(filesPendingUpload);
         await FilesDB.instance.insertMultiple(filesPendingUpload);
       }
       if (files.isNotEmpty) {

lib/ui/viewer/file/file_icons_widget.dart

@@ -8,7 +8,6 @@ class ThumbnailPlaceHolder extends StatelessWidget {
 
   @override
   Widget build(BuildContext context) {
-    // debugPrint("building placeHolder for thumbnail");
     return Container(
       alignment: Alignment.center,
       color: Theme.of(context).colorScheme.galleryThumbBackgroundColor,

lib/ui/viewer/gallery/device_folder_page.dart

@@ -81,10 +81,12 @@ class BackupConfigurationHeaderWidget extends StatefulWidget {
 
 class _BackupConfigurationHeaderWidgetState
     extends State<BackupConfigurationHeaderWidget> {
-  bool isBackedUp;
+  bool _isBackedUp;
+
   @override
   void initState() {
-    isBackedUp = widget.devicePathCollection.sync;
+    _isBackedUp = widget.devicePathCollection.sync;
+    super.initState();
   }
 
   @override
@@ -96,7 +98,7 @@ class _BackupConfigurationHeaderWidgetState
       child: Row(
         mainAxisAlignment: MainAxisAlignment.spaceBetween,
         children: [
-          isBackedUp
+          _isBackedUp
               ? const Text("Backup enabled")
               : Text(
                   "Backup disabled",
@@ -108,12 +110,12 @@ class _BackupConfigurationHeaderWidgetState
                   ),
                 ),
           Switch(
-            value: isBackedUp,
+            value: _isBackedUp,
             onChanged: (value) async {
               await FilesDB.instance.updateDevicePathSyncStatus(
                 {widget.devicePathCollection.id: value},
               );
-              isBackedUp = value;
+              _isBackedUp = value;
               setState(() {});
               Bus.instance.fire(BackupFoldersUpdatedEvent());
             },

lib/utils/file_uploader.dart

@@ -447,10 +447,10 @@ class FileUploader {
   }
 
   /*
-  // _mapToExistingUpload links the current file to be uploaded with the
-  // existing files. If the link is successful, it returns true other false.
-   When false, we should go ahead and re-upload or update the file
-    It performs following checks:
+  _mapToExistingUpload links the fileToUpload with the existing uploaded
+  files. if the link is successful, it returns true otherwise false.
+  When false, we should go ahead and re-upload or update the file.
+  It performs following checks:
     a) Uploaded file with same localID and destination collection. Delete the
      fileToUpload entry
     b) Uploaded file in destination collection but with missing localID.
@@ -469,32 +469,27 @@ class FileUploader {
     File fileToUpload,
     int toCollectionID,
   ) async {
-    if (fileToUpload.uploadedFileID != -1 &&
-        fileToUpload.uploadedFileID != null) {
-      _logger.warning('file is already uploaded, skipping mapping logic');
+    if (fileToUpload.uploadedFileID != null) {
+      _logger.severe(
+        'Critical: file is already uploaded, skipped mapping',
+      );
       return false;
     }
-    final List<String> hash = [mediaUploadData.fileHash];
-    if (fileToUpload.fileType == FileType.livePhoto) {
-      hash.add(mediaUploadData.zipHash);
-    }
-    final List<File> existingFiles =
+    final List<File> existingUploadedFiles =
         await FilesDB.instance.getUploadedFilesWithHashes(
-      hash,
+      mediaUploadData.hashData,
       fileToUpload.fileType,
       Configuration.instance.getUserID(),
     );
-    if (existingFiles?.isEmpty ?? true) {
+    if (existingUploadedFiles?.isEmpty ?? true) {
       return false;
     } else {
       debugPrint("Found some matches");
     }
     // case a
-    final File sameLocalSameCollection = existingFiles.firstWhere(
-      (element) =>
-          element.uploadedFileID != -1 &&
-          element.collectionID == toCollectionID &&
-          element.localID == fileToUpload.localID,
+    final File sameLocalSameCollection = existingUploadedFiles.firstWhere(
+      (e) =>
+          e.collectionID == toCollectionID && e.localID == fileToUpload.localID,
       orElse: () => null,
     );
     if (sameLocalSameCollection != null) {
@@ -503,16 +498,14 @@ class FileUploader {
         "\n existing: ${sameLocalSameCollection.tag()}",
       );
       // should delete the fileToUploadEntry
-      FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID);
+      await FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID);
       return true;
     }
 
     // case b
-    final File fileMissingLocalButSameCollection = existingFiles.firstWhere(
-      (element) =>
-          element.uploadedFileID != -1 &&
-          element.collectionID == toCollectionID &&
-          element.localID == null,
+    final File fileMissingLocalButSameCollection =
+        existingUploadedFiles.firstWhere(
+      (e) => e.collectionID == toCollectionID && e.localID == null,
       orElse: () => null,
     );
     if (fileMissingLocalButSameCollection != null) {
@@ -529,10 +522,9 @@ class FileUploader {
     }
 
     // case c and d
-    final File fileExistsButDifferentCollection = existingFiles.firstWhere(
-      (element) =>
-          element.uploadedFileID != -1 &&
-          element.collectionID != toCollectionID,
+    final File fileExistsButDifferentCollection =
+        existingUploadedFiles.firstWhere(
+      (e) => e.collectionID != toCollectionID,
       orElse: () => null,
     );
     if (fileExistsButDifferentCollection != null) {
@@ -542,7 +534,10 @@ class FileUploader {
       );
       await CollectionsService.instance
           .linkLocalFileToExistingUploadedFileInAnotherCollection(
-              toCollectionID, fileToUpload, fileExistsButDifferentCollection);
+        toCollectionID,
+        localFileToUpload: fileToUpload,
+        existingUploadedFile: fileExistsButDifferentCollection,
+      );
       return true;
     }
     // case e

lib/utils/file_uploader_util.dart

@@ -21,25 +21,31 @@ import 'package:video_thumbnail/video_thumbnail.dart';
 
 final _logger = Logger("FileUtil");
 const kMaximumThumbnailCompressionAttempts = 2;
+const kLivePhotoHashSeparator = ':';
 
 class MediaUploadData {
   final io.File sourceFile;
   final Uint8List thumbnail;
   final bool isDeleted;
-  // presents the hash for the original video or image file.
-  // for livePhotos, fileHash represents the image hash value
-  final String fileHash;
-  final String liveVideoHash;
-  final String zipHash;
+  final FileHashData hashData;
 
   MediaUploadData(
     this.sourceFile,
     this.thumbnail,
-    this.isDeleted, {
-    this.fileHash,
-    this.liveVideoHash,
-    this.zipHash,
-  });
+    this.isDeleted,
+    this.hashData,
+  );
+}
+
+class FileHashData {
+  // For livePhotos, the fileHash value will be imageHash:videoHash
+  final String fileHash;
+
+  // zipHash is used to take care of existing live photo uploads from older
+  // mobile clients
+  String zipHash;
+
+  FileHashData(this.fileHash, {this.zipHash});
 }
 
 Future<MediaUploadData> getUploadDataFromEnteFile(ente.File file) async {
@@ -54,7 +60,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
   io.File sourceFile;
   Uint8List thumbnailData;
   bool isDeleted;
-  String fileHash, livePhotoVideoHash, zipHash;
+  String fileHash, zipHash;
 
   // The timeouts are to safeguard against https://github.com/CaiJingLong/flutter_photo_manager/issues/467
   final asset = await file
@@ -97,7 +103,10 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
       _logger.severe(errMsg);
       throw InvalidFileUploadState(errMsg);
     }
-    livePhotoVideoHash = Sodium.bin2base64(await CryptoUtil.getHash(videoUrl));
+    final String livePhotoVideoHash =
+        Sodium.bin2base64(await CryptoUtil.getHash(videoUrl));
+    // imgHash:vidHash
+    fileHash = '$fileHash$kLivePhotoHashSeparator$livePhotoVideoHash';
     final tempPath = Configuration.instance.getTempDirectory();
     // .elp -> ente live photo
     final livePhotoPath = tempPath + file.generatedID.toString() + ".elp";
@@ -138,9 +147,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
     sourceFile,
     thumbnailData,
     isDeleted,
-    fileHash: fileHash,
-    liveVideoHash: livePhotoVideoHash,
-    zipHash: zipHash,
+    FileHashData(fileHash, zipHash: zipHash),
   );
 }
 
@@ -170,7 +177,13 @@ Future<MediaUploadData> _getMediaUploadDataFromAppCache(ente.File file) async {
   }
   try {
     thumbnailData = await getThumbnailFromInAppCacheFile(file);
-    return MediaUploadData(sourceFile, thumbnailData, isDeleted);
+    final fileHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
+    return MediaUploadData(
+      sourceFile,
+      thumbnailData,
+      isDeleted,
+      FileHashData(fileHash),
+    );
   } catch (e, s) {
     _logger.severe("failed to generate thumbnail", e, s);
     throw InvalidFileError(