Merge pull request #415 from ente-io/reupload_hash_check

[Part-0] Rewrite Sync: Use hash to avoid duplicate uploads

lib/db/file_migration_db.dart → lib/db/file_updation_db.dart

@@ -1,33 +1,57 @@
 import 'dart:io';
 
+import 'package:flutter/foundation.dart';
 import 'package:logging/logging.dart';
 import 'package:path/path.dart';
 import 'package:path_provider/path_provider.dart';
 import 'package:sqflite/sqflite.dart';
+import 'package:sqflite_migration/sqflite_migration.dart';
 
-class FilesMigrationDB {
+class FileUpdationDB {
   static const _databaseName = "ente.files_migration.db";
-  static const _databaseVersion = 1;
-  static final Logger _logger = Logger((FilesMigrationDB).toString());
-  static const tableName = 're_upload_tracker';
+  static final Logger _logger = Logger((FileUpdationDB).toString());
 
+  static const tableName = 're_upload_tracker';
   static const columnLocalID = 'local_id';
+  static const columnReason = 'reason';
+  static const missingLocation = 'missing_location';
+  static const modificationTimeUpdated = 'modificationTimeUpdated';
+
+  // SQL code to create the database table
+  static List<String> _createTable() {
+    return [
+      ''' 
+      CREATE TABLE $tableName (
+      $columnLocalID TEXT NOT NULL,
+      UNIQUE($columnLocalID)
+      ); 
+      ''',
+    ];
+  }
 
-  Future _onCreate(Database db, int version) async {
-    await db.execute(
+  static List<String> addReasonColumn() {
+    return [
       '''
-        CREATE TABLE $tableName (
-        $columnLocalID TEXT NOT NULL,
-          UNIQUE($columnLocalID)
-        );
+        ALTER TABLE $tableName ADD COLUMN $columnReason TEXT;
       ''',
-    );
+      '''
+        UPDATE $tableName SET $columnReason = '$missingLocation';
+      ''',
+    ];
   }
 
-  FilesMigrationDB._privateConstructor();
+  static final initializationScript = [..._createTable()];
+  static final migrationScripts = [
+    ...addReasonColumn(),
+  ];
+  final dbConfig = MigrationConfig(
+    initializationScript: initializationScript,
+    migrationScripts: migrationScripts,
+  );
 
-  static final FilesMigrationDB instance =
-      FilesMigrationDB._privateConstructor();
+  FileUpdationDB._privateConstructor();
+
+  static final FileUpdationDB instance = FileUpdationDB._privateConstructor();
 
   // only have a single app-wide reference to the database
   static Future<Database> _dbFuture;
@@ -40,13 +64,11 @@ class FilesMigrationDB {
 
   // this opens the database (and creates it if it doesn't exist)
   Future<Database> _initDatabase() async {
-    final Directory documentsDirectory = await getApplicationDocumentsDirectory();
+    final Directory documentsDirectory =
+        await getApplicationDocumentsDirectory();
     final String path = join(documentsDirectory.path, _databaseName);
-    return await openDatabase(
-      path,
-      version: _databaseVersion,
-      onCreate: _onCreate,
-    );
+    debugPrint("DB path " + path);
+    return await openDatabaseWithMigration(path, dbConfig);
   }
 
   Future<void> clearTable() async {
@@ -54,7 +76,10 @@ class FilesMigrationDB {
     await db.delete(tableName);
   }
 
-  Future<void> insertMultiple(List<String> fileLocalIDs) async {
+  Future<void> insertMultiple(
+    List<String> fileLocalIDs,
+    String reason,
+  ) async {
     final startTime = DateTime.now();
     final db = await instance.database;
     var batch = db.batch();
@@ -67,7 +92,7 @@ class FilesMigrationDB {
       }
       batch.insert(
         tableName,
-        _getRowForReUploadTable(localID),
+        _getRowForReUploadTable(localID, reason),
         conflictAlgorithm: ConflictAlgorithm.replace,
       );
       batchCounter++;
@@ -84,22 +109,35 @@ class FilesMigrationDB {
     );
   }
 
-  Future<int> deleteByLocalIDs(List<String> localIDs) async {
+  Future<void> deleteByLocalIDs(List<String> localIDs, String reason) async {
+    if (localIDs.isEmpty) {
+      return;
+    }
     String inParam = "";
     for (final localID in localIDs) {
       inParam += "'" + localID + "',";
     }
     inParam = inParam.substring(0, inParam.length - 1);
     final db = await instance.database;
-    return await db.delete(
-      tableName,
-      where: '$columnLocalID IN (${localIDs.join(', ')})',
+    await db.rawQuery(
+      '''
+      DELETE FROM $tableName
+      WHERE $columnLocalID IN ($inParam) AND $columnReason = '$reason';
+    ''',
     );
   }
 
-  Future<List<String>> getLocalIDsForPotentialReUpload(int limit) async {
+  Future<List<String>> getLocalIDsForPotentialReUpload(
+    int limit,
+    String reason,
+  ) async {
     final db = await instance.database;
-    final rows = await db.query(tableName, limit: limit);
+    String whereClause = '$columnReason = "$reason"';
+    final rows = await db.query(
+      tableName,
+      limit: limit,
+      where: whereClause,
+    );
     final result = <String>[];
     for (final row in rows) {
       result.add(row[columnLocalID]);
@@ -107,10 +145,11 @@ class FilesMigrationDB {
     return result;
   }
 
-  Map<String, dynamic> _getRowForReUploadTable(String localID) {
+  Map<String, dynamic> _getRowForReUploadTable(String localID, String reason) {
     assert(localID != null);
     final row = <String, dynamic>{};
     row[columnLocalID] = localID;
+    row[columnReason] = reason;
     return row;
   }
 }

lib/db/files_db.dart

@@ -10,6 +10,7 @@ import 'package:photos/models/file_type.dart';
 import 'package:photos/models/location.dart';
 import 'package:photos/models/magic_metadata.dart';
 import 'package:photos/services/feature_flag_service.dart';
+import 'package:photos/utils/file_uploader_util.dart';
 import 'package:sqflite/sqflite.dart';
 import 'package:sqflite_migration/sqflite_migration.dart';
 
@@ -848,6 +849,31 @@ class FilesDB {
     }
   }
 
+  Future<List<File>> getUploadedFilesWithHashes(
+    FileHashData hashData,
+    FileType fileType,
+    int ownerID,
+  ) async {
+    String inParam = "'${hashData.fileHash}'";
+    if (fileType == FileType.livePhoto && hashData.zipHash != null) {
+      inParam += ",'${hashData.zipHash}'";
+    }
+
+    final db = await instance.database;
+    final rows = await db.query(
+      table,
+      where: '($columnUploadedFileID != NULL OR $columnUploadedFileID != -1) '
+          'AND $columnOwnerID = ? AND $columnFileType ='
+          ' ? '
+          'AND $columnHash IN ($inParam)',
+      whereArgs: [
+        ownerID,
+        getInt(fileType),
+      ],
+    );
+    return _convertToFiles(rows);
+  }
+
   Future<int> update(File file) async {
     final db = await instance.database;
     return await db.update(
@@ -877,6 +903,15 @@ class FilesDB {
     );
   }
 
+  Future<int> deleteByGeneratedID(int genID) async {
+    final db = await instance.database;
+    return db.delete(
+      table,
+      where: '$columnGeneratedID =?',
+      whereArgs: [genID],
+    );
+  }
+
   Future<int> deleteMultipleUploadedFiles(List<int> uploadedFileIDs) async {
     final db = await instance.database;
     return await db.delete(
@@ -1089,27 +1124,11 @@ class FilesDB {
     return collectionMap.values.toList();
   }
 
-  Future<File> getLastModifiedFileInCollection(int collectionID) async {
-    final db = await instance.database;
-    final rows = await db.query(
-      table,
-      where: '$columnCollectionID = ?',
-      whereArgs: [collectionID],
-      orderBy: '$columnUpdationTime DESC',
-      limit: 1,
-    );
-    if (rows.isNotEmpty) {
-      return _getFileFromRow(rows[0]);
-    } else {
-      return null;
-    }
-  }
-
   Future<Map<String, int>> getFileCountInDeviceFolders() async {
     final db = await instance.database;
     final rows = await db.rawQuery(
       '''
-      SELECT COUNT($columnGeneratedID) as count, $columnDeviceFolder
+      SELECT COUNT(DISTINCT($columnLocalID)) as count, $columnDeviceFolder
       FROM $table
       WHERE $columnLocalID IS NOT NULL
       GROUP BY $columnDeviceFolder

lib/main.dart

@@ -19,7 +19,7 @@ import 'package:photos/services/app_lifecycle_service.dart';
 import 'package:photos/services/billing_service.dart';
 import 'package:photos/services/collections_service.dart';
 import 'package:photos/services/feature_flag_service.dart';
-import 'package:photos/services/file_migration_service.dart';
+import 'package:photos/services/local_file_update_service.dart';
 import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/services/memories_service.dart';
 import 'package:photos/services/notification_service.dart';
@@ -141,7 +141,7 @@ Future<void> _init(bool isBackground, {String via = ''}) async {
   await SyncService.instance.init();
   await MemoriesService.instance.init();
   await LocalSettings.instance.init();
-  await FileMigrationService.instance.init();
+  await LocalFileUpdateService.instance.init();
   await SearchService.instance.init();
   if (Platform.isIOS) {
     PushService.instance.init().then((_) {

lib/models/file.dart

@@ -1,6 +1,3 @@
-import 'dart:io' as io;
-
-import 'package:flutter_sodium/flutter_sodium.dart';
 import 'package:path/path.dart';
 import 'package:photo_manager/photo_manager.dart';
 import 'package:photos/core/configuration.dart';
@@ -10,8 +7,8 @@ import 'package:photos/models/file_type.dart';
 import 'package:photos/models/location.dart';
 import 'package:photos/models/magic_metadata.dart';
 import 'package:photos/services/feature_flag_service.dart';
-import 'package:photos/utils/crypto_util.dart';
 import 'package:photos/utils/exif_util.dart';
+import 'package:photos/utils/file_uploader_util.dart';
 
 class File extends EnteFile {
   int generatedID;
@@ -56,7 +53,9 @@ class File extends EnteFile {
 
   set pubMagicMetadata(val) => _pubMmd = val;
 
-  static const kCurrentMetadataVersion = 1;
+  // in Version 1, live photo hash is stored as zip's hash.
+  // in V2: LivePhoto hash is stored as imgHash:vidHash
+  static const kCurrentMetadataVersion = 2;
 
   File();
 
@@ -136,10 +135,21 @@ class File extends EnteFile {
     duration = metadata["duration"] ?? 0;
     exif = metadata["exif"];
     hash = metadata["hash"];
+    // handle past live photos upload from web client
+    if (hash == null &&
+        fileType == FileType.livePhoto &&
+        metadata.containsKey('imgHash') &&
+        metadata.containsKey('vidHash')) {
+      // convert to imgHash:vidHash
+      hash =
+          '${metadata['imgHash']}$kLivePhotoHashSeparator${metadata['vidHash']}';
+    }
     metadataVersion = metadata["version"] ?? 0;
   }
 
-  Future<Map<String, dynamic>> getMetadataForUpload(io.File sourceFile) async {
+  Future<Map<String, dynamic>> getMetadataForUpload(
+    MediaUploadData mediaUploadData,
+  ) async {
     final asset = await getAsset();
     // asset can be null for files shared to app
     if (asset != null) {
@@ -149,12 +159,13 @@ class File extends EnteFile {
       }
     }
     if (fileType == FileType.image) {
-      final exifTime = await getCreationTimeFromEXIF(sourceFile);
+      final exifTime =
+          await getCreationTimeFromEXIF(mediaUploadData.sourceFile);
       if (exifTime != null) {
         creationTime = exifTime.microsecondsSinceEpoch;
       }
     }
-    hash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
+    hash = mediaUploadData.hashData?.fileHash;
     return getMetadata();
   }
 

lib/services/collections_service.dart

@@ -630,6 +630,51 @@ class CollectionsService {
     }
   }
 
+  Future<File> linkLocalFileToExistingUploadedFileInAnotherCollection(
+    int destCollectionID, {
+    @required File localFileToUpload,
+    @required File existingUploadedFile,
+  }) async {
+    final params = <String, dynamic>{};
+    params["collectionID"] = destCollectionID;
+    params["files"] = [];
+    final int uploadedFileID = existingUploadedFile.uploadedFileID;
+
+    // encrypt the fileKey with destination collection's key
+    final fileKey = decryptFileKey(existingUploadedFile);
+    final encryptedKeyData =
+        CryptoUtil.encryptSync(fileKey, getCollectionKey(destCollectionID));
+
+    localFileToUpload.encryptedKey =
+        Sodium.bin2base64(encryptedKeyData.encryptedData);
+    localFileToUpload.keyDecryptionNonce =
+        Sodium.bin2base64(encryptedKeyData.nonce);
+
+    params["files"].add(
+      CollectionFileItem(
+        uploadedFileID,
+        localFileToUpload.encryptedKey,
+        localFileToUpload.keyDecryptionNonce,
+      ).toMap(),
+    );
+
+    try {
+      await _dio.post(
+        Configuration.instance.getHttpEndpoint() + "/collections/add-files",
+        data: params,
+        options: Options(
+          headers: {"X-Auth-Token": Configuration.instance.getToken()},
+        ),
+      );
+      localFileToUpload.collectionID = destCollectionID;
+      localFileToUpload.uploadedFileID = uploadedFileID;
+      await _filesDB.insertMultiple([localFileToUpload]);
+      return localFileToUpload;
+    } catch (e) {
+      rethrow;
+    }
+  }
+
   Future<void> restore(int toCollectionID, List<File> files) async {
     final params = <String, dynamic>{};
     params["collectionID"] = toCollectionID;

lib/services/file_migration_service.dart

@@ -1,137 +0,0 @@
-import 'dart:async';
-import 'dart:core';
-import 'dart:io';
-
-import 'package:logging/logging.dart';
-import 'package:photo_manager/photo_manager.dart';
-import 'package:photos/db/file_migration_db.dart';
-import 'package:photos/db/files_db.dart';
-import 'package:shared_preferences/shared_preferences.dart';
-
-class FileMigrationService {
-  FilesDB _filesDB;
-  FilesMigrationDB _filesMigrationDB;
-  SharedPreferences _prefs;
-  Logger _logger;
-  static const isLocationMigrationComplete = "fm_isLocationMigrationComplete";
-  static const isLocalImportDone = "fm_IsLocalImportDone";
-  Completer<void> _existingMigration;
-
-  FileMigrationService._privateConstructor() {
-    _logger = Logger((FileMigrationService).toString());
-    _filesDB = FilesDB.instance;
-    _filesMigrationDB = FilesMigrationDB.instance;
-  }
-
-  Future<void> init() async {
-    _prefs = await SharedPreferences.getInstance();
-  }
-
-  static FileMigrationService instance =
-      FileMigrationService._privateConstructor();
-
-  Future<bool> _markLocationMigrationAsCompleted() async {
-    _logger.info('marking migration as completed');
-    return _prefs.setBool(isLocationMigrationComplete, true);
-  }
-
-  bool isLocationMigrationCompleted() {
-    return _prefs.get(isLocationMigrationComplete) ?? false;
-  }
-
-  Future<void> runMigration() async {
-    if (_existingMigration != null) {
-      _logger.info("migration is already in progress, skipping");
-      return _existingMigration.future;
-    }
-    _logger.info("start migration");
-    _existingMigration = Completer<void>();
-    try {
-      await _runMigrationForFilesWithMissingLocation();
-      _existingMigration.complete();
-      _existingMigration = null;
-    } catch (e, s) {
-      _logger.severe('failed to perform migration', e, s);
-      _existingMigration.complete();
-      _existingMigration = null;
-    }
-  }
-
-  Future<void> _runMigrationForFilesWithMissingLocation() async {
-    if (!Platform.isAndroid) {
-      return;
-    }
-    // migration only needs to run if Android API Level is 29 or higher
-    final int version = int.parse(await PhotoManager.systemVersion());
-    final bool isMigrationRequired = version >= 29;
-    if (isMigrationRequired) {
-      await _importLocalFilesForMigration();
-      final sTime = DateTime.now().microsecondsSinceEpoch;
-      bool hasData = true;
-      const int limitInBatch = 100;
-      while (hasData) {
-        final localIDsToProcess = await _filesMigrationDB
-            .getLocalIDsForPotentialReUpload(limitInBatch);
-        if (localIDsToProcess.isEmpty) {
-          hasData = false;
-        } else {
-          await _checkAndMarkFilesForReUpload(localIDsToProcess);
-        }
-      }
-      final eTime = DateTime.now().microsecondsSinceEpoch;
-      final d = Duration(microseconds: eTime - sTime);
-      _logger.info(
-        'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds',
-      );
-    }
-    await _markLocationMigrationAsCompleted();
-  }
-
-  Future<void> _checkAndMarkFilesForReUpload(
-    List<String> localIDsToProcess,
-  ) async {
-    _logger.info("files to process ${localIDsToProcess.length}");
-    final localIDsWithLocation = <String>[];
-    for (var localID in localIDsToProcess) {
-      bool hasLocation = false;
-      try {
-        final assetEntity = await AssetEntity.fromId(localID);
-        if (assetEntity == null) {
-          continue;
-        }
-        final latLng = await assetEntity.latlngAsync();
-        if ((latLng.longitude ?? 0.0) != 0.0 ||
-            (latLng.longitude ?? 0.0) != 0.0) {
-          _logger.finest(
-            'found lat/long ${latLng.longitude}/${latLng.longitude} for  ${assetEntity.title} ${assetEntity.relativePath} with id : $localID',
-          );
-          hasLocation = true;
-        }
-      } catch (e, s) {
-        _logger.severe('failed to get asset entity with id $localID', e, s);
-      }
-      if (hasLocation) {
-        localIDsWithLocation.add(localID);
-      }
-    }
-    _logger.info('marking ${localIDsWithLocation.length} files for re-upload');
-    await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation);
-    await _filesMigrationDB.deleteByLocalIDs(localIDsToProcess);
-  }
-
-  Future<void> _importLocalFilesForMigration() async {
-    if (_prefs.containsKey(isLocalImportDone)) {
-      return;
-    }
-    final sTime = DateTime.now().microsecondsSinceEpoch;
-    _logger.info('importing files without location info');
-    final fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation();
-    await _filesMigrationDB.insertMultiple(fileLocalIDs);
-    final eTime = DateTime.now().microsecondsSinceEpoch;
-    final d = Duration(microseconds: eTime - sTime);
-    _logger.info(
-      'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds',
-    );
-    _prefs.setBool(isLocalImportDone, true);
-  }
-}

lib/services/local_file_update_service.dart

@@ -0,0 +1,240 @@
+import 'dart:async';
+import 'dart:core';
+import 'dart:io';
+
+import 'package:flutter/foundation.dart';
+import 'package:logging/logging.dart';
+import 'package:photo_manager/photo_manager.dart';
+import 'package:photos/db/file_updation_db.dart';
+import 'package:photos/db/files_db.dart';
+import 'package:photos/models/file.dart' as ente;
+import 'package:photos/utils/file_uploader_util.dart';
+import 'package:shared_preferences/shared_preferences.dart';
+
+// LocalFileUpdateService tracks all the potential local file IDs which have
+// changed/modified on the device and needed to be uploaded again.
+class LocalFileUpdateService {
+  FilesDB _filesDB;
+  FileUpdationDB _fileUpdationDB;
+  SharedPreferences _prefs;
+  Logger _logger;
+  static const isLocationMigrationComplete = "fm_isLocationMigrationComplete";
+  static const isLocalImportDone = "fm_IsLocalImportDone";
+  Completer<void> _existingMigration;
+
+  LocalFileUpdateService._privateConstructor() {
+    _logger = Logger((LocalFileUpdateService).toString());
+    _filesDB = FilesDB.instance;
+    _fileUpdationDB = FileUpdationDB.instance;
+  }
+
+  Future<void> init() async {
+    _prefs = await SharedPreferences.getInstance();
+  }
+
+  static LocalFileUpdateService instance =
+      LocalFileUpdateService._privateConstructor();
+
+  Future<bool> _markLocationMigrationAsCompleted() async {
+    _logger.info('marking migration as completed');
+    return _prefs.setBool(isLocationMigrationComplete, true);
+  }
+
+  bool isLocationMigrationCompleted() {
+    return _prefs.get(isLocationMigrationComplete) ?? false;
+  }
+
+  Future<void> markUpdatedFilesForReUpload() async {
+    if (_existingMigration != null) {
+      _logger.info("migration is already in progress, skipping");
+      return _existingMigration.future;
+    }
+    _existingMigration = Completer<void>();
+    try {
+      if (!isLocationMigrationCompleted() && Platform.isAndroid) {
+        _logger.info("start migration for missing location");
+        await _runMigrationForFilesWithMissingLocation();
+      }
+      await _markFilesWhichAreActuallyUpdated();
+    } catch (e, s) {
+      _logger.severe('failed to perform migration', e, s);
+    } finally {
+      _existingMigration?.complete();
+      _existingMigration = null;
+    }
+  }
+
+  // This method analyses all of local files for which the file
+  // modification/update time was changed. It checks if the existing fileHash
+  // is different from the hash of uploaded file. If fileHash are different,
+  // then it marks the file for file update.
+  Future<void> _markFilesWhichAreActuallyUpdated() async {
+    final sTime = DateTime.now().microsecondsSinceEpoch;
+    bool hasData = true;
+    const int limitInBatch = 100;
+    while (hasData) {
+      final localIDsToProcess =
+          await _fileUpdationDB.getLocalIDsForPotentialReUpload(
+        limitInBatch,
+        FileUpdationDB.modificationTimeUpdated,
+      );
+      if (localIDsToProcess.isEmpty) {
+        hasData = false;
+      } else {
+        await _checkAndMarkFilesWithDifferentHashForFileUpdate(
+          localIDsToProcess,
+        );
+      }
+    }
+    final eTime = DateTime.now().microsecondsSinceEpoch;
+    final d = Duration(microseconds: eTime - sTime);
+    _logger.info(
+      '_markFilesWhichAreActuallyUpdated migration completed in ${d.inSeconds.toString()} seconds',
+    );
+  }
+
+  Future<void> _checkAndMarkFilesWithDifferentHashForFileUpdate(
+    List<String> localIDsToProcess,
+  ) async {
+    _logger.info("files to process ${localIDsToProcess.length} for reupload");
+    List<ente.File> localFiles =
+        (await FilesDB.instance.getLocalFiles(localIDsToProcess));
+    Set<String> processedIDs = {};
+    for (ente.File file in localFiles) {
+      if (processedIDs.contains(file.localID)) {
+        continue;
+      }
+      MediaUploadData uploadData;
+      try {
+        uploadData = await getUploadData(file);
+        if (uploadData != null &&
+            uploadData.hashData != null &&
+            file.hash != null &&
+            (file.hash == uploadData.hashData.fileHash ||
+                file.hash == uploadData.hashData.zipHash)) {
+          _logger.info("Skip file update as hash matched ${file.tag()}");
+        } else {
+          _logger.info(
+            "Marking for file update as hash did not match ${file.tag()}",
+          );
+          await FilesDB.instance.updateUploadedFile(
+            file.localID,
+            file.title,
+            file.location,
+            file.creationTime,
+            file.modificationTime,
+            null,
+          );
+        }
+        processedIDs.add(file.localID);
+      } catch (e) {
+        _logger.severe("Failed to get file uploadData", e);
+      } finally {}
+    }
+    debugPrint("Deleting files ${processedIDs.length}");
+    await _fileUpdationDB.deleteByLocalIDs(
+      processedIDs.toList(),
+      FileUpdationDB.modificationTimeUpdated,
+    );
+  }
+
+  Future<MediaUploadData> getUploadData(ente.File file) async {
+    final mediaUploadData = await getUploadDataFromEnteFile(file);
+    // delete the file from app's internal cache if it was copied to app
+    // for upload. Shared Media should only be cleared when the upload
+    // succeeds.
+    if (Platform.isIOS &&
+        mediaUploadData != null &&
+        mediaUploadData.sourceFile != null) {
+      await mediaUploadData.sourceFile.delete();
+    }
+    return mediaUploadData;
+  }
+
+  Future<void> _runMigrationForFilesWithMissingLocation() async {
+    if (!Platform.isAndroid) {
+      return;
+    }
+    // migration only needs to run if Android API Level is 29 or higher
+    final int version = int.parse(await PhotoManager.systemVersion());
+    bool isMigrationRequired = version >= 29;
+    if (isMigrationRequired) {
+      await _importLocalFilesForMigration();
+      final sTime = DateTime.now().microsecondsSinceEpoch;
+      bool hasData = true;
+      const int limitInBatch = 100;
+      while (hasData) {
+        var localIDsToProcess =
+            await _fileUpdationDB.getLocalIDsForPotentialReUpload(
+          limitInBatch,
+          FileUpdationDB.missingLocation,
+        );
+        if (localIDsToProcess.isEmpty) {
+          hasData = false;
+        } else {
+          await _checkAndMarkFilesWithLocationForReUpload(localIDsToProcess);
+        }
+      }
+      final eTime = DateTime.now().microsecondsSinceEpoch;
+      final d = Duration(microseconds: eTime - sTime);
+      _logger.info(
+        'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds',
+      );
+    }
+    await _markLocationMigrationAsCompleted();
+  }
+
+  Future<void> _checkAndMarkFilesWithLocationForReUpload(
+    List<String> localIDsToProcess,
+  ) async {
+    _logger.info("files to process ${localIDsToProcess.length}");
+    var localIDsWithLocation = <String>[];
+    for (var localID in localIDsToProcess) {
+      bool hasLocation = false;
+      try {
+        var assetEntity = await AssetEntity.fromId(localID);
+        if (assetEntity == null) {
+          continue;
+        }
+        var latLng = await assetEntity.latlngAsync();
+        if ((latLng.longitude ?? 0.0) != 0.0 ||
+            (latLng.longitude ?? 0.0) != 0.0) {
+          _logger.finest(
+            'found lat/long ${latLng.longitude}/${latLng.longitude} for  ${assetEntity.title} ${assetEntity.relativePath} with id : $localID',
+          );
+          hasLocation = true;
+        }
+      } catch (e, s) {
+        _logger.severe('failed to get asset entity with id $localID', e, s);
+      }
+      if (hasLocation) {
+        localIDsWithLocation.add(localID);
+      }
+    }
+    _logger.info('marking ${localIDsWithLocation.length} files for re-upload');
+    await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation);
+    await _fileUpdationDB.deleteByLocalIDs(
+      localIDsToProcess,
+      FileUpdationDB.missingLocation,
+    );
+  }
+
+  Future<void> _importLocalFilesForMigration() async {
+    if (_prefs.containsKey(isLocalImportDone)) {
+      return;
+    }
+    final sTime = DateTime.now().microsecondsSinceEpoch;
+    _logger.info('importing files without location info');
+    var fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation();
+    await _fileUpdationDB.insertMultiple(
+      fileLocalIDs,
+      FileUpdationDB.missingLocation,
+    );
+    final eTime = DateTime.now().microsecondsSinceEpoch;
+    final d = Duration(microseconds: eTime - sTime);
+    _logger.info(
+      'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds',
+    );
+    await _prefs.setBool(isLocalImportDone, true);
+  }
+}

lib/services/local_sync_service.dart

@@ -6,6 +6,7 @@ import 'package:logging/logging.dart';
 import 'package:photo_manager/photo_manager.dart';
 import 'package:photos/core/configuration.dart';
 import 'package:photos/core/event_bus.dart';
+import 'package:photos/db/file_updation_db.dart';
 import 'package:photos/db/files_db.dart';
 import 'package:photos/events/local_photos_updated_event.dart';
 import 'package:photos/events/sync_status_update_event.dart';
@@ -31,6 +32,7 @@ class LocalSyncService {
   // Adding `_2` as a suffic to pull files that were earlier ignored due to permission errors
   // See https://github.com/CaiJingLong/flutter_photo_manager/issues/589
   static const kInvalidFileIDsKey = "invalid_file_ids_2";
+
   LocalSyncService._privateConstructor();
 
   static final LocalSyncService instance =
@@ -243,17 +245,15 @@ class LocalSyncService {
           updatedFiles.length.toString() + " local files were updated.",
         );
       }
+
+      final List<String> updatedLocalIDs = [];
       for (final file in updatedFiles) {
-        await captureUpdateLogs(file);
-        await _db.updateUploadedFile(
-          file.localID,
-          file.title,
-          file.location,
-          file.creationTime,
-          file.modificationTime,
-          null,
-        );
+        updatedLocalIDs.add(file.localID);
       }
+      await FileUpdationDB.instance.insertMultiple(
+        updatedLocalIDs,
+        FileUpdationDB.modificationTimeUpdated,
+      );
       final List<File> allFiles = [];
       allFiles.addAll(files);
       files.removeWhere((file) => existingLocalFileIDs.contains(file.localID));
@@ -265,32 +265,6 @@ class LocalSyncService {
     await _prefs.setInt(kDbUpdationTimeKey, toTime);
   }
 
-  // _captureUpdateLogs is a helper method to log details
-  // about the file which is being marked for re-upload
-  Future<void> captureUpdateLogs(File file) async {
-    _logger.info(
-      're-upload locally updated file ${file.toString()}',
-    );
-    try {
-      if (Platform.isIOS) {
-        final assetEntity = await AssetEntity.fromId(file.localID);
-        if (assetEntity != null) {
-          final isLocallyAvailable =
-              await assetEntity.isLocallyAvailable(isOrigin: true);
-          _logger.info(
-            're-upload asset ${file.toString()} with localAvailableFlag '
-            '$isLocallyAvailable and fav ${assetEntity.isFavorite}',
-          );
-        } else {
-          _logger
-              .info('re-upload failed to fetch assetInfo ${file.toString()}');
-        }
-      }
-    } catch (ignore) {
-      //ignore
-    }
-  }
-
   void _updatePathsToBackup(List<File> files) {
     if (Configuration.instance.hasSelectedAllFoldersForBackup()) {
       final pathsToBackup = Configuration.instance.getPathsToBackUp();

lib/services/remote_sync_service.dart

@@ -16,9 +16,8 @@ import 'package:photos/models/file.dart';
 import 'package:photos/models/file_type.dart';
 import 'package:photos/services/app_lifecycle_service.dart';
 import 'package:photos/services/collections_service.dart';
-import 'package:photos/services/feature_flag_service.dart';
-import 'package:photos/services/file_migration_service.dart';
 import 'package:photos/services/ignored_files_service.dart';
+import 'package:photos/services/local_file_update_service.dart';
 import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/services/trash_sync_service.dart';
 import 'package:photos/utils/diff_fetcher.dart';
@@ -32,8 +31,8 @@ class RemoteSyncService {
   final _uploader = FileUploader.instance;
   final _collectionsService = CollectionsService.instance;
   final _diffFetcher = DiffFetcher();
-  final FileMigrationService _fileMigrationService =
-      FileMigrationService.instance;
+  final LocalFileUpdateService _localFileUpdateService =
+      LocalFileUpdateService.instance;
   int _completedUploads = 0;
   SharedPreferences _prefs;
   Completer<void> _existingSync;
@@ -134,10 +133,8 @@ class RemoteSyncService {
     if (!_hasReSynced()) {
       await _markReSyncAsDone();
     }
-    if (FeatureFlagService.instance.enableMissingLocationMigration() &&
-        !_fileMigrationService.isLocationMigrationCompleted()) {
-      _fileMigrationService.runMigration();
-    }
+
+    unawaited(_localFileUpdateService.markUpdatedFilesForReUpload());
   }
 
   Future<void> _syncUpdatedCollections(bool silently) async {

lib/utils/file_uploader.dart

@@ -7,6 +7,7 @@ import 'dart:typed_data';
 
 import 'package:connectivity/connectivity.dart';
 import 'package:dio/dio.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter_sodium/flutter_sodium.dart';
 import 'package:logging/logging.dart';
 import 'package:path/path.dart';
@@ -31,6 +32,7 @@ import 'package:photos/utils/crypto_util.dart';
 import 'package:photos/utils/file_download_util.dart';
 import 'package:photos/utils/file_uploader_util.dart';
 import 'package:shared_preferences/shared_preferences.dart';
+import 'package:tuple/tuple.dart';
 
 class FileUploader {
   static const kMaximumConcurrentUploads = 4;
@@ -108,6 +110,8 @@ class FileUploader {
     });
   }
 
+  // upload future will return null as File when the file entry is deleted
+  // locally because it's already present in the destination collection.
   Future<File> upload(File file, int collectionID) {
     // If the file hasn't been queued yet, queue it
     _totalCountInUploadSession++;
@@ -125,10 +129,25 @@ class FileUploader {
       _totalCountInUploadSession--;
       return item.completer.future;
     }
-
+    debugPrint(
+      "Wait on another upload on same local ID to finish before "
+      "adding it to new collection",
+    );
     // Else wait for the existing upload to complete,
     // and add it to the relevant collection
     return item.completer.future.then((uploadedFile) {
+      // If the fileUploader completer returned null,
+      _logger.info(
+        "original upload completer resolved, try adding the file to another "
+        "collection",
+      );
+      if (uploadedFile == null) {
+        /* todo: handle this case, ideally during next sync the localId
+          should be uploaded to this collection ID
+         */
+        _logger.severe('unexpected upload state');
+        return null;
+      }
       return CollectionsService.instance
           .addToCollection(collectionID, [uploadedFile]).then((aVoid) {
         return uploadedFile;
@@ -136,61 +155,6 @@ class FileUploader {
     });
   }
 
-  Future<File> forceUpload(File file, int collectionID) async {
-    _logger.info(
-      "Force uploading " +
-          file.toString() +
-          " into collection " +
-          collectionID.toString(),
-    );
-    _totalCountInUploadSession++;
-    // If the file hasn't been queued yet, ez.
-    if (!_queue.containsKey(file.localID)) {
-      final completer = Completer<File>();
-      _queue[file.localID] = FileUploadItem(
-        file,
-        collectionID,
-        completer,
-        status: UploadStatus.inProgress,
-      );
-      _encryptAndUploadFileToCollection(file, collectionID, forcedUpload: true);
-      return completer.future;
-    }
-    var item = _queue[file.localID];
-    // If the file is being uploaded right now, wait and proceed
-    if (item.status == UploadStatus.inProgress ||
-        item.status == UploadStatus.inBackground) {
-      _totalCountInUploadSession--;
-      final uploadedFile = await item.completer.future;
-      if (uploadedFile.collectionID == collectionID) {
-        // Do nothing
-      } else {
-        await CollectionsService.instance
-            .addToCollection(collectionID, [uploadedFile]);
-      }
-      return uploadedFile;
-    } else {
-      // If the file is yet to be processed,
-      // 1. Set the status to in_progress
-      // 2. Force upload the file
-      // 3. Add to the relevant collection
-      item = _queue[file.localID];
-      item.status = UploadStatus.inProgress;
-      final uploadedFile = await _encryptAndUploadFileToCollection(
-        file,
-        collectionID,
-        forcedUpload: true,
-      );
-      if (item.collectionID == collectionID) {
-        return uploadedFile;
-      } else {
-        await CollectionsService.instance
-            .addToCollection(item.collectionID, [uploadedFile]);
-        return uploadedFile;
-      }
-    }
-  }
-
   int getCurrentSessionUploadCount() {
     return _totalCountInUploadSession;
   }
@@ -319,6 +283,7 @@ class FileUploader {
         fileOnDisk.updationTime != -1 &&
         fileOnDisk.collectionID == collectionID;
     if (wasAlreadyUploaded) {
+      debugPrint("File is already uploaded ${fileOnDisk.tag()}");
       return fileOnDisk;
     }
 
@@ -362,6 +327,7 @@ class FileUploader {
           rethrow;
         }
       }
+
       Uint8List key;
       final bool isUpdatedFile =
           file.uploadedFileID != null && file.updationTime == -1;
@@ -370,6 +336,22 @@ class FileUploader {
         key = decryptFileKey(file);
       } else {
         key = null;
+        // check if the file is already uploaded and can be mapped to existing
+        // uploaded file. If map is found, it also returns the corresponding
+        // mapped or update file entry.
+        final result = await _mapToExistingUploadWithSameHash(
+          mediaUploadData,
+          file,
+          collectionID,
+        );
+        final isMappedToExistingUpload = result.item1;
+        if (isMappedToExistingUpload) {
+          debugPrint(
+            "File success mapped to existing uploaded ${file.toString()}",
+          );
+          // return the mapped file
+          return result.item2;
+        }
       }
 
       if (io.File(encryptedFilePath).existsSync()) {
@@ -399,8 +381,7 @@ class FileUploader {
       final fileUploadURL = await _getUploadURL();
       final String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
 
-      final metadata =
-          await file.getMetadataForUpload(mediaUploadData.sourceFile);
+      final metadata = await file.getMetadataForUpload(mediaUploadData);
       final encryptedMetadataData = await CryptoUtil.encryptChaCha(
         utf8.encode(jsonEncode(metadata)),
         fileAttributes.key,
@@ -476,23 +457,141 @@ class FileUploader {
       }
       rethrow;
     } finally {
-      if (mediaUploadData != null && mediaUploadData.sourceFile != null) {
-        // delete the file from app's internal cache if it was copied to app
-        // for upload. Shared Media should only be cleared when the upload
-        // succeeds.
-        if (io.Platform.isIOS ||
-            (uploadCompleted && file.isSharedMediaToAppSandbox())) {
-          await mediaUploadData.sourceFile.delete();
-        }
-      }
-      if (io.File(encryptedFilePath).existsSync()) {
-        await io.File(encryptedFilePath).delete();
-      }
-      if (io.File(encryptedThumbnailPath).existsSync()) {
-        await io.File(encryptedThumbnailPath).delete();
+      await _onUploadDone(
+        mediaUploadData,
+        uploadCompleted,
+        file,
+        encryptedFilePath,
+        encryptedThumbnailPath,
+      );
+    }
+  }
+
+  /*
+  _mapToExistingUpload links the fileToUpload with the existing uploaded
+  files. if the link is successful, it returns true otherwise false.
+  When false, we should go ahead and re-upload or update the file.
+  It performs following checks:
+    a) Uploaded file with same localID and destination collection. Delete the
+     fileToUpload entry
+    b) Uploaded file in destination collection but with missing localID.
+     Update the localID for uploadedFile and delete the fileToUpload entry
+    c) A uploaded file exist with same localID but in a different collection.
+    or
+    d) Uploaded file in different collection but missing localID.
+    For both c and d, perform add to collection operation.
+    e) File already exists but different localID. Re-upload
+    In case the existing files already have local identifier, which is
+    different from the {fileToUpload}, then most probably device has
+    duplicate files.
+  */
+  Future<Tuple2<bool, File>> _mapToExistingUploadWithSameHash(
+    MediaUploadData mediaUploadData,
+    File fileToUpload,
+    int toCollectionID,
+  ) async {
+    if (fileToUpload.uploadedFileID != null) {
+      // ideally this should never happen, but because the code below this case
+      // can do unexpected mapping, we are adding this additional check
+      _logger.severe(
+        'Critical: file is already uploaded, skipped mapping',
+      );
+      return Tuple2(false, fileToUpload);
+    }
+
+    final List<File> existingUploadedFiles =
+        await FilesDB.instance.getUploadedFilesWithHashes(
+      mediaUploadData.hashData,
+      fileToUpload.fileType,
+      Configuration.instance.getUserID(),
+    );
+    if (existingUploadedFiles?.isEmpty ?? true) {
+      // continueUploading this file
+      return Tuple2(false, fileToUpload);
+    } else {
+      debugPrint("Found some matches");
+    }
+    // case a
+    final File sameLocalSameCollection = existingUploadedFiles.firstWhere(
+      (e) =>
+          e.collectionID == toCollectionID && e.localID == fileToUpload.localID,
+      orElse: () => null,
+    );
+    if (sameLocalSameCollection != null) {
+      debugPrint(
+        "sameLocalSameCollection: \n toUpload  ${fileToUpload.tag()} "
+        "\n existing: ${sameLocalSameCollection.tag()}",
+      );
+      // should delete the fileToUploadEntry
+      await FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID);
+      return Tuple2(true, sameLocalSameCollection);
+    }
+
+    // case b
+    final File fileMissingLocalButSameCollection =
+        existingUploadedFiles.firstWhere(
+      (e) => e.collectionID == toCollectionID && e.localID == null,
+      orElse: () => null,
+    );
+    if (fileMissingLocalButSameCollection != null) {
+      // update the local id of the existing file and delete the fileToUpload
+      // entry
+      debugPrint(
+        "fileMissingLocalButSameCollection: \n toUpload  ${fileToUpload.tag()} "
+        "\n existing: ${fileMissingLocalButSameCollection.tag()}",
+      );
+      fileMissingLocalButSameCollection.localID = fileToUpload.localID;
+      await FilesDB.instance.insert(fileMissingLocalButSameCollection);
+      await FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID);
+      return Tuple2(true, fileMissingLocalButSameCollection);
+    }
+
+    // case c and d
+    final File fileExistsButDifferentCollection =
+        existingUploadedFiles.firstWhere(
+      (e) => e.collectionID != toCollectionID,
+      orElse: () => null,
+    );
+    if (fileExistsButDifferentCollection != null) {
+      debugPrint(
+        "fileExistsButDifferentCollection: \n toUpload  ${fileToUpload.tag()} "
+        "\n existing: ${fileExistsButDifferentCollection.tag()}",
+      );
+      final linkedFile = await CollectionsService.instance
+          .linkLocalFileToExistingUploadedFileInAnotherCollection(
+        toCollectionID,
+        localFileToUpload: fileToUpload,
+        existingUploadedFile: fileExistsButDifferentCollection,
+      );
+      return Tuple2(true, linkedFile);
+    }
+    // case e
+    return Tuple2(false, fileToUpload);
+  }
+
+  Future<void> _onUploadDone(
+    MediaUploadData mediaUploadData,
+    bool uploadCompleted,
+    File file,
+    String encryptedFilePath,
+    String encryptedThumbnailPath,
+  ) async {
+    if (mediaUploadData != null && mediaUploadData.sourceFile != null) {
+      // delete the file from app's internal cache if it was copied to app
+      // for upload. Shared Media should only be cleared when the upload
+      // succeeds.
+      if (io.Platform.isIOS ||
+          (uploadCompleted && file.isSharedMediaToAppSandbox())) {
+        await mediaUploadData.sourceFile.delete();
       }
-      await _uploadLocks.releaseLock(file.localID, _processType.toString());
     }
+    if (io.File(encryptedFilePath).existsSync()) {
+      await io.File(encryptedFilePath).delete();
+    }
+    if (io.File(encryptedThumbnailPath).existsSync()) {
+      await io.File(encryptedThumbnailPath).delete();
+    }
+    await _uploadLocks.releaseLock(file.localID, _processType.toString());
   }
 
   Future _onInvalidFileError(File file, InvalidFileError e) async {

lib/utils/file_uploader_util.dart

@@ -3,6 +3,7 @@ import 'dart:io' as io;
 import 'dart:typed_data';
 
 import 'package:archive/archive_io.dart';
+import 'package:flutter_sodium/flutter_sodium.dart';
 import 'package:logging/logging.dart';
 import 'package:motionphoto/motionphoto.dart';
 import 'package:path/path.dart';
@@ -14,18 +15,37 @@ import 'package:photos/core/errors.dart';
 import 'package:photos/models/file.dart' as ente;
 import 'package:photos/models/file_type.dart';
 import 'package:photos/models/location.dart';
+import 'package:photos/utils/crypto_util.dart';
 import 'package:photos/utils/file_util.dart';
 import 'package:video_thumbnail/video_thumbnail.dart';
 
 final _logger = Logger("FileUtil");
 const kMaximumThumbnailCompressionAttempts = 2;
+const kLivePhotoHashSeparator = ':';
 
 class MediaUploadData {
   final io.File sourceFile;
   final Uint8List thumbnail;
   final bool isDeleted;
+  final FileHashData hashData;
 
-  MediaUploadData(this.sourceFile, this.thumbnail, this.isDeleted);
+  MediaUploadData(
+    this.sourceFile,
+    this.thumbnail,
+    this.isDeleted,
+    this.hashData,
+  );
+}
+
+class FileHashData {
+  // For livePhotos, the fileHash value will be imageHash:videoHash
+  final String fileHash;
+
+  // zipHash is used to take care of existing live photo uploads from older
+  // mobile clients
+  String zipHash;
+
+  FileHashData(this.fileHash, {this.zipHash});
 }
 
 Future<MediaUploadData> getUploadDataFromEnteFile(ente.File file) async {
@@ -40,6 +60,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
   io.File sourceFile;
   Uint8List thumbnailData;
   bool isDeleted;
+  String fileHash, zipHash;
 
   // The timeouts are to safeguard against https://github.com/CaiJingLong/flutter_photo_manager/issues/467
   final asset = await file
@@ -72,6 +93,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
 
   // h4ck to fetch location data if missing (thank you Android Q+) lazily only during uploads
   await _decorateEnteFileData(file, asset);
+  fileHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
 
   if (file.fileType == FileType.livePhoto && io.Platform.isIOS) {
     final io.File videoUrl = await Motionphoto.getLivePhotoFile(file.localID);
@@ -81,6 +103,10 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
       _logger.severe(errMsg);
       throw InvalidFileUploadState(errMsg);
     }
+    String livePhotoVideoHash =
+        Sodium.bin2base64(await CryptoUtil.getHash(videoUrl));
+    // imgHash:vidHash
+    fileHash = '$fileHash$kLivePhotoHashSeparator$livePhotoVideoHash';
     final tempPath = Configuration.instance.getTempDirectory();
     // .elp -> ente live photo
     final livePhotoPath = tempPath + file.generatedID.toString() + ".elp";
@@ -96,6 +122,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
     }
     // new sourceFile which needs to be uploaded
     sourceFile = io.File(livePhotoPath);
+    zipHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
   }
 
   thumbnailData = await asset.thumbnailDataWithSize(
@@ -116,7 +143,12 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
   }
 
   isDeleted = asset == null || !(await asset.exists);
-  return MediaUploadData(sourceFile, thumbnailData, isDeleted);
+  return MediaUploadData(
+    sourceFile,
+    thumbnailData,
+    isDeleted,
+    FileHashData(fileHash, zipHash: zipHash),
+  );
 }
 
 Future<void> _decorateEnteFileData(ente.File file, AssetEntity asset) async {
@@ -128,7 +160,7 @@ Future<void> _decorateEnteFileData(ente.File file, AssetEntity asset) async {
   }
 
   if (file.title == null || file.title.isEmpty) {
-    _logger.severe("Title was missing");
+    _logger.warning("Title was missing ${file.tag()}");
     file.title = await asset.titleAsync;
   }
 }
@@ -145,7 +177,13 @@ Future<MediaUploadData> _getMediaUploadDataFromAppCache(ente.File file) async {
   }
   try {
     thumbnailData = await getThumbnailFromInAppCacheFile(file);
-    return MediaUploadData(sourceFile, thumbnailData, isDeleted);
+    final fileHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
+    return MediaUploadData(
+      sourceFile,
+      thumbnailData,
+      isDeleted,
+      FileHashData(fileHash),
+    );
   } catch (e, s) {
     _logger.severe("failed to generate thumbnail", e, s);
     throw InvalidFileError(