diff --git a/lib/db/file_migration_db.dart b/lib/db/file_updation_db.dart similarity index 51% rename from lib/db/file_migration_db.dart rename to lib/db/file_updation_db.dart index 43e9f7b04..02ff00c7a 100644 --- a/lib/db/file_migration_db.dart +++ b/lib/db/file_updation_db.dart @@ -1,33 +1,57 @@ import 'dart:io'; +import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; import 'package:sqflite/sqflite.dart'; +import 'package:sqflite_migration/sqflite_migration.dart'; -class FilesMigrationDB { +class FileUpdationDB { static const _databaseName = "ente.files_migration.db"; - static const _databaseVersion = 1; - static final Logger _logger = Logger((FilesMigrationDB).toString()); + static final Logger _logger = Logger((FileUpdationDB).toString()); + static const tableName = 're_upload_tracker'; - static const columnLocalID = 'local_id'; + static const columnReason = 'reason'; + static const missingLocation = 'missing_location'; + static const modificationTimeUpdated = 'modificationTimeUpdated'; - Future _onCreate(Database db, int version) async { - await db.execute( - ''' - CREATE TABLE $tableName ( - $columnLocalID TEXT NOT NULL, - UNIQUE($columnLocalID) - ); + // SQL code to create the database table + static List _createTable() { + return [ + ''' + CREATE TABLE $tableName ( + $columnLocalID TEXT NOT NULL, + UNIQUE($columnLocalID) + ); ''', - ); + ]; } - FilesMigrationDB._privateConstructor(); + static List addReasonColumn() { + return [ + ''' + ALTER TABLE $tableName ADD COLUMN $columnReason TEXT; + ''', + ''' + UPDATE $tableName SET $columnReason = '$missingLocation'; + ''', + ]; + } - static final FilesMigrationDB instance = - FilesMigrationDB._privateConstructor(); + static final initializationScript = [..._createTable()]; + static final migrationScripts = [ + ...addReasonColumn(), + ]; + final dbConfig = MigrationConfig( + initializationScript: initializationScript, + migrationScripts: migrationScripts, + ); + + FileUpdationDB._privateConstructor(); + + static final FileUpdationDB instance = FileUpdationDB._privateConstructor(); // only have a single app-wide reference to the database static Future _dbFuture; @@ -40,13 +64,11 @@ class FilesMigrationDB { // this opens the database (and creates it if it doesn't exist) Future _initDatabase() async { - final Directory documentsDirectory = await getApplicationDocumentsDirectory(); + final Directory documentsDirectory = + await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); - return await openDatabase( - path, - version: _databaseVersion, - onCreate: _onCreate, - ); + debugPrint("DB path " + path); + return await openDatabaseWithMigration(path, dbConfig); } Future clearTable() async { @@ -54,7 +76,10 @@ class FilesMigrationDB { await db.delete(tableName); } - Future insertMultiple(List fileLocalIDs) async { + Future insertMultiple( + List fileLocalIDs, + String reason, + ) async { final startTime = DateTime.now(); final db = await instance.database; var batch = db.batch(); @@ -67,7 +92,7 @@ class FilesMigrationDB { } batch.insert( tableName, - _getRowForReUploadTable(localID), + _getRowForReUploadTable(localID, reason), conflictAlgorithm: ConflictAlgorithm.replace, ); batchCounter++; @@ -84,22 +109,35 @@ class FilesMigrationDB { ); } - Future deleteByLocalIDs(List localIDs) async { + Future deleteByLocalIDs(List localIDs, String reason) async { + if (localIDs.isEmpty) { + return; + } String inParam = ""; for (final localID in localIDs) { inParam += "'" + localID + "',"; } inParam = inParam.substring(0, inParam.length - 1); final db = await instance.database; - return await db.delete( - tableName, - where: '$columnLocalID IN (${localIDs.join(', ')})', + await db.rawQuery( + ''' + DELETE FROM $tableName + WHERE $columnLocalID IN ($inParam) AND $columnReason = '$reason'; + ''', ); } - Future> getLocalIDsForPotentialReUpload(int limit) async { + Future> getLocalIDsForPotentialReUpload( + int limit, + String reason, + ) async { final db = await instance.database; - final rows = await db.query(tableName, limit: limit); + String whereClause = '$columnReason = "$reason"'; + final rows = await db.query( + tableName, + limit: limit, + where: whereClause, + ); final result = []; for (final row in rows) { result.add(row[columnLocalID]); @@ -107,10 +145,11 @@ class FilesMigrationDB { return result; } - Map _getRowForReUploadTable(String localID) { + Map _getRowForReUploadTable(String localID, String reason) { assert(localID != null); final row = {}; row[columnLocalID] = localID; + row[columnReason] = reason; return row; } } diff --git a/lib/db/files_db.dart b/lib/db/files_db.dart index 4f93e23e7..698965e0d 100644 --- a/lib/db/files_db.dart +++ b/lib/db/files_db.dart @@ -10,6 +10,7 @@ import 'package:photos/models/file_type.dart'; import 'package:photos/models/location.dart'; import 'package:photos/models/magic_metadata.dart'; import 'package:photos/services/feature_flag_service.dart'; +import 'package:photos/utils/file_uploader_util.dart'; import 'package:sqflite/sqflite.dart'; import 'package:sqflite_migration/sqflite_migration.dart'; @@ -848,6 +849,31 @@ class FilesDB { } } + Future> getUploadedFilesWithHashes( + FileHashData hashData, + FileType fileType, + int ownerID, + ) async { + String inParam = "'${hashData.fileHash}'"; + if (fileType == FileType.livePhoto && hashData.zipHash != null) { + inParam += ",'${hashData.zipHash}'"; + } + + final db = await instance.database; + final rows = await db.query( + table, + where: '($columnUploadedFileID != NULL OR $columnUploadedFileID != -1) ' + 'AND $columnOwnerID = ? AND $columnFileType =' + ' ? ' + 'AND $columnHash IN ($inParam)', + whereArgs: [ + ownerID, + getInt(fileType), + ], + ); + return _convertToFiles(rows); + } + Future update(File file) async { final db = await instance.database; return await db.update( @@ -877,6 +903,15 @@ class FilesDB { ); } + Future deleteByGeneratedID(int genID) async { + final db = await instance.database; + return db.delete( + table, + where: '$columnGeneratedID =?', + whereArgs: [genID], + ); + } + Future deleteMultipleUploadedFiles(List uploadedFileIDs) async { final db = await instance.database; return await db.delete( @@ -1089,27 +1124,11 @@ class FilesDB { return collectionMap.values.toList(); } - Future getLastModifiedFileInCollection(int collectionID) async { - final db = await instance.database; - final rows = await db.query( - table, - where: '$columnCollectionID = ?', - whereArgs: [collectionID], - orderBy: '$columnUpdationTime DESC', - limit: 1, - ); - if (rows.isNotEmpty) { - return _getFileFromRow(rows[0]); - } else { - return null; - } - } - Future> getFileCountInDeviceFolders() async { final db = await instance.database; final rows = await db.rawQuery( ''' - SELECT COUNT($columnGeneratedID) as count, $columnDeviceFolder + SELECT COUNT(DISTINCT($columnLocalID)) as count, $columnDeviceFolder FROM $table WHERE $columnLocalID IS NOT NULL GROUP BY $columnDeviceFolder diff --git a/lib/main.dart b/lib/main.dart index ff88f8e1b..d8ae140ed 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -19,7 +19,7 @@ import 'package:photos/services/app_lifecycle_service.dart'; import 'package:photos/services/billing_service.dart'; import 'package:photos/services/collections_service.dart'; import 'package:photos/services/feature_flag_service.dart'; -import 'package:photos/services/file_migration_service.dart'; +import 'package:photos/services/local_file_update_service.dart'; import 'package:photos/services/local_sync_service.dart'; import 'package:photos/services/memories_service.dart'; import 'package:photos/services/notification_service.dart'; @@ -141,7 +141,7 @@ Future _init(bool isBackground, {String via = ''}) async { await SyncService.instance.init(); await MemoriesService.instance.init(); await LocalSettings.instance.init(); - await FileMigrationService.instance.init(); + await LocalFileUpdateService.instance.init(); await SearchService.instance.init(); if (Platform.isIOS) { PushService.instance.init().then((_) { diff --git a/lib/models/file.dart b/lib/models/file.dart index 139735f9b..7089fed24 100644 --- a/lib/models/file.dart +++ b/lib/models/file.dart @@ -1,6 +1,3 @@ -import 'dart:io' as io; - -import 'package:flutter_sodium/flutter_sodium.dart'; import 'package:path/path.dart'; import 'package:photo_manager/photo_manager.dart'; import 'package:photos/core/configuration.dart'; @@ -10,8 +7,8 @@ import 'package:photos/models/file_type.dart'; import 'package:photos/models/location.dart'; import 'package:photos/models/magic_metadata.dart'; import 'package:photos/services/feature_flag_service.dart'; -import 'package:photos/utils/crypto_util.dart'; import 'package:photos/utils/exif_util.dart'; +import 'package:photos/utils/file_uploader_util.dart'; class File extends EnteFile { int generatedID; @@ -56,7 +53,9 @@ class File extends EnteFile { set pubMagicMetadata(val) => _pubMmd = val; - static const kCurrentMetadataVersion = 1; + // in Version 1, live photo hash is stored as zip's hash. + // in V2: LivePhoto hash is stored as imgHash:vidHash + static const kCurrentMetadataVersion = 2; File(); @@ -136,10 +135,21 @@ class File extends EnteFile { duration = metadata["duration"] ?? 0; exif = metadata["exif"]; hash = metadata["hash"]; + // handle past live photos upload from web client + if (hash == null && + fileType == FileType.livePhoto && + metadata.containsKey('imgHash') && + metadata.containsKey('vidHash')) { + // convert to imgHash:vidHash + hash = + '${metadata['imgHash']}$kLivePhotoHashSeparator${metadata['vidHash']}'; + } metadataVersion = metadata["version"] ?? 0; } - Future> getMetadataForUpload(io.File sourceFile) async { + Future> getMetadataForUpload( + MediaUploadData mediaUploadData, + ) async { final asset = await getAsset(); // asset can be null for files shared to app if (asset != null) { @@ -149,12 +159,13 @@ class File extends EnteFile { } } if (fileType == FileType.image) { - final exifTime = await getCreationTimeFromEXIF(sourceFile); + final exifTime = + await getCreationTimeFromEXIF(mediaUploadData.sourceFile); if (exifTime != null) { creationTime = exifTime.microsecondsSinceEpoch; } } - hash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile)); + hash = mediaUploadData.hashData?.fileHash; return getMetadata(); } diff --git a/lib/services/collections_service.dart b/lib/services/collections_service.dart index 707b06b3d..7ab31a606 100644 --- a/lib/services/collections_service.dart +++ b/lib/services/collections_service.dart @@ -630,6 +630,51 @@ class CollectionsService { } } + Future linkLocalFileToExistingUploadedFileInAnotherCollection( + int destCollectionID, { + @required File localFileToUpload, + @required File existingUploadedFile, + }) async { + final params = {}; + params["collectionID"] = destCollectionID; + params["files"] = []; + final int uploadedFileID = existingUploadedFile.uploadedFileID; + + // encrypt the fileKey with destination collection's key + final fileKey = decryptFileKey(existingUploadedFile); + final encryptedKeyData = + CryptoUtil.encryptSync(fileKey, getCollectionKey(destCollectionID)); + + localFileToUpload.encryptedKey = + Sodium.bin2base64(encryptedKeyData.encryptedData); + localFileToUpload.keyDecryptionNonce = + Sodium.bin2base64(encryptedKeyData.nonce); + + params["files"].add( + CollectionFileItem( + uploadedFileID, + localFileToUpload.encryptedKey, + localFileToUpload.keyDecryptionNonce, + ).toMap(), + ); + + try { + await _dio.post( + Configuration.instance.getHttpEndpoint() + "/collections/add-files", + data: params, + options: Options( + headers: {"X-Auth-Token": Configuration.instance.getToken()}, + ), + ); + localFileToUpload.collectionID = destCollectionID; + localFileToUpload.uploadedFileID = uploadedFileID; + await _filesDB.insertMultiple([localFileToUpload]); + return localFileToUpload; + } catch (e) { + rethrow; + } + } + Future restore(int toCollectionID, List files) async { final params = {}; params["collectionID"] = toCollectionID; diff --git a/lib/services/file_migration_service.dart b/lib/services/file_migration_service.dart deleted file mode 100644 index f71ac986d..000000000 --- a/lib/services/file_migration_service.dart +++ /dev/null @@ -1,137 +0,0 @@ -import 'dart:async'; -import 'dart:core'; -import 'dart:io'; - -import 'package:logging/logging.dart'; -import 'package:photo_manager/photo_manager.dart'; -import 'package:photos/db/file_migration_db.dart'; -import 'package:photos/db/files_db.dart'; -import 'package:shared_preferences/shared_preferences.dart'; - -class FileMigrationService { - FilesDB _filesDB; - FilesMigrationDB _filesMigrationDB; - SharedPreferences _prefs; - Logger _logger; - static const isLocationMigrationComplete = "fm_isLocationMigrationComplete"; - static const isLocalImportDone = "fm_IsLocalImportDone"; - Completer _existingMigration; - - FileMigrationService._privateConstructor() { - _logger = Logger((FileMigrationService).toString()); - _filesDB = FilesDB.instance; - _filesMigrationDB = FilesMigrationDB.instance; - } - - Future init() async { - _prefs = await SharedPreferences.getInstance(); - } - - static FileMigrationService instance = - FileMigrationService._privateConstructor(); - - Future _markLocationMigrationAsCompleted() async { - _logger.info('marking migration as completed'); - return _prefs.setBool(isLocationMigrationComplete, true); - } - - bool isLocationMigrationCompleted() { - return _prefs.get(isLocationMigrationComplete) ?? false; - } - - Future runMigration() async { - if (_existingMigration != null) { - _logger.info("migration is already in progress, skipping"); - return _existingMigration.future; - } - _logger.info("start migration"); - _existingMigration = Completer(); - try { - await _runMigrationForFilesWithMissingLocation(); - _existingMigration.complete(); - _existingMigration = null; - } catch (e, s) { - _logger.severe('failed to perform migration', e, s); - _existingMigration.complete(); - _existingMigration = null; - } - } - - Future _runMigrationForFilesWithMissingLocation() async { - if (!Platform.isAndroid) { - return; - } - // migration only needs to run if Android API Level is 29 or higher - final int version = int.parse(await PhotoManager.systemVersion()); - final bool isMigrationRequired = version >= 29; - if (isMigrationRequired) { - await _importLocalFilesForMigration(); - final sTime = DateTime.now().microsecondsSinceEpoch; - bool hasData = true; - const int limitInBatch = 100; - while (hasData) { - final localIDsToProcess = await _filesMigrationDB - .getLocalIDsForPotentialReUpload(limitInBatch); - if (localIDsToProcess.isEmpty) { - hasData = false; - } else { - await _checkAndMarkFilesForReUpload(localIDsToProcess); - } - } - final eTime = DateTime.now().microsecondsSinceEpoch; - final d = Duration(microseconds: eTime - sTime); - _logger.info( - 'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds', - ); - } - await _markLocationMigrationAsCompleted(); - } - - Future _checkAndMarkFilesForReUpload( - List localIDsToProcess, - ) async { - _logger.info("files to process ${localIDsToProcess.length}"); - final localIDsWithLocation = []; - for (var localID in localIDsToProcess) { - bool hasLocation = false; - try { - final assetEntity = await AssetEntity.fromId(localID); - if (assetEntity == null) { - continue; - } - final latLng = await assetEntity.latlngAsync(); - if ((latLng.longitude ?? 0.0) != 0.0 || - (latLng.longitude ?? 0.0) != 0.0) { - _logger.finest( - 'found lat/long ${latLng.longitude}/${latLng.longitude} for ${assetEntity.title} ${assetEntity.relativePath} with id : $localID', - ); - hasLocation = true; - } - } catch (e, s) { - _logger.severe('failed to get asset entity with id $localID', e, s); - } - if (hasLocation) { - localIDsWithLocation.add(localID); - } - } - _logger.info('marking ${localIDsWithLocation.length} files for re-upload'); - await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation); - await _filesMigrationDB.deleteByLocalIDs(localIDsToProcess); - } - - Future _importLocalFilesForMigration() async { - if (_prefs.containsKey(isLocalImportDone)) { - return; - } - final sTime = DateTime.now().microsecondsSinceEpoch; - _logger.info('importing files without location info'); - final fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation(); - await _filesMigrationDB.insertMultiple(fileLocalIDs); - final eTime = DateTime.now().microsecondsSinceEpoch; - final d = Duration(microseconds: eTime - sTime); - _logger.info( - 'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds', - ); - _prefs.setBool(isLocalImportDone, true); - } -} diff --git a/lib/services/local_file_update_service.dart b/lib/services/local_file_update_service.dart new file mode 100644 index 000000000..fd70d4fa3 --- /dev/null +++ b/lib/services/local_file_update_service.dart @@ -0,0 +1,240 @@ +import 'dart:async'; +import 'dart:core'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:logging/logging.dart'; +import 'package:photo_manager/photo_manager.dart'; +import 'package:photos/db/file_updation_db.dart'; +import 'package:photos/db/files_db.dart'; +import 'package:photos/models/file.dart' as ente; +import 'package:photos/utils/file_uploader_util.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +// LocalFileUpdateService tracks all the potential local file IDs which have +// changed/modified on the device and needed to be uploaded again. +class LocalFileUpdateService { + FilesDB _filesDB; + FileUpdationDB _fileUpdationDB; + SharedPreferences _prefs; + Logger _logger; + static const isLocationMigrationComplete = "fm_isLocationMigrationComplete"; + static const isLocalImportDone = "fm_IsLocalImportDone"; + Completer _existingMigration; + + LocalFileUpdateService._privateConstructor() { + _logger = Logger((LocalFileUpdateService).toString()); + _filesDB = FilesDB.instance; + _fileUpdationDB = FileUpdationDB.instance; + } + + Future init() async { + _prefs = await SharedPreferences.getInstance(); + } + + static LocalFileUpdateService instance = + LocalFileUpdateService._privateConstructor(); + + Future _markLocationMigrationAsCompleted() async { + _logger.info('marking migration as completed'); + return _prefs.setBool(isLocationMigrationComplete, true); + } + + bool isLocationMigrationCompleted() { + return _prefs.get(isLocationMigrationComplete) ?? false; + } + + Future markUpdatedFilesForReUpload() async { + if (_existingMigration != null) { + _logger.info("migration is already in progress, skipping"); + return _existingMigration.future; + } + _existingMigration = Completer(); + try { + if (!isLocationMigrationCompleted() && Platform.isAndroid) { + _logger.info("start migration for missing location"); + await _runMigrationForFilesWithMissingLocation(); + } + await _markFilesWhichAreActuallyUpdated(); + } catch (e, s) { + _logger.severe('failed to perform migration', e, s); + } finally { + _existingMigration?.complete(); + _existingMigration = null; + } + } + + // This method analyses all of local files for which the file + // modification/update time was changed. It checks if the existing fileHash + // is different from the hash of uploaded file. If fileHash are different, + // then it marks the file for file update. + Future _markFilesWhichAreActuallyUpdated() async { + final sTime = DateTime.now().microsecondsSinceEpoch; + bool hasData = true; + const int limitInBatch = 100; + while (hasData) { + final localIDsToProcess = + await _fileUpdationDB.getLocalIDsForPotentialReUpload( + limitInBatch, + FileUpdationDB.modificationTimeUpdated, + ); + if (localIDsToProcess.isEmpty) { + hasData = false; + } else { + await _checkAndMarkFilesWithDifferentHashForFileUpdate( + localIDsToProcess, + ); + } + } + final eTime = DateTime.now().microsecondsSinceEpoch; + final d = Duration(microseconds: eTime - sTime); + _logger.info( + '_markFilesWhichAreActuallyUpdated migration completed in ${d.inSeconds.toString()} seconds', + ); + } + + Future _checkAndMarkFilesWithDifferentHashForFileUpdate( + List localIDsToProcess, + ) async { + _logger.info("files to process ${localIDsToProcess.length} for reupload"); + List localFiles = + (await FilesDB.instance.getLocalFiles(localIDsToProcess)); + Set processedIDs = {}; + for (ente.File file in localFiles) { + if (processedIDs.contains(file.localID)) { + continue; + } + MediaUploadData uploadData; + try { + uploadData = await getUploadData(file); + if (uploadData != null && + uploadData.hashData != null && + file.hash != null && + (file.hash == uploadData.hashData.fileHash || + file.hash == uploadData.hashData.zipHash)) { + _logger.info("Skip file update as hash matched ${file.tag()}"); + } else { + _logger.info( + "Marking for file update as hash did not match ${file.tag()}", + ); + await FilesDB.instance.updateUploadedFile( + file.localID, + file.title, + file.location, + file.creationTime, + file.modificationTime, + null, + ); + } + processedIDs.add(file.localID); + } catch (e) { + _logger.severe("Failed to get file uploadData", e); + } finally {} + } + debugPrint("Deleting files ${processedIDs.length}"); + await _fileUpdationDB.deleteByLocalIDs( + processedIDs.toList(), + FileUpdationDB.modificationTimeUpdated, + ); + } + + Future getUploadData(ente.File file) async { + final mediaUploadData = await getUploadDataFromEnteFile(file); + // delete the file from app's internal cache if it was copied to app + // for upload. Shared Media should only be cleared when the upload + // succeeds. + if (Platform.isIOS && + mediaUploadData != null && + mediaUploadData.sourceFile != null) { + await mediaUploadData.sourceFile.delete(); + } + return mediaUploadData; + } + + Future _runMigrationForFilesWithMissingLocation() async { + if (!Platform.isAndroid) { + return; + } + // migration only needs to run if Android API Level is 29 or higher + final int version = int.parse(await PhotoManager.systemVersion()); + bool isMigrationRequired = version >= 29; + if (isMigrationRequired) { + await _importLocalFilesForMigration(); + final sTime = DateTime.now().microsecondsSinceEpoch; + bool hasData = true; + const int limitInBatch = 100; + while (hasData) { + var localIDsToProcess = + await _fileUpdationDB.getLocalIDsForPotentialReUpload( + limitInBatch, + FileUpdationDB.missingLocation, + ); + if (localIDsToProcess.isEmpty) { + hasData = false; + } else { + await _checkAndMarkFilesWithLocationForReUpload(localIDsToProcess); + } + } + final eTime = DateTime.now().microsecondsSinceEpoch; + final d = Duration(microseconds: eTime - sTime); + _logger.info( + 'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds', + ); + } + await _markLocationMigrationAsCompleted(); + } + + Future _checkAndMarkFilesWithLocationForReUpload( + List localIDsToProcess, + ) async { + _logger.info("files to process ${localIDsToProcess.length}"); + var localIDsWithLocation = []; + for (var localID in localIDsToProcess) { + bool hasLocation = false; + try { + var assetEntity = await AssetEntity.fromId(localID); + if (assetEntity == null) { + continue; + } + var latLng = await assetEntity.latlngAsync(); + if ((latLng.longitude ?? 0.0) != 0.0 || + (latLng.longitude ?? 0.0) != 0.0) { + _logger.finest( + 'found lat/long ${latLng.longitude}/${latLng.longitude} for ${assetEntity.title} ${assetEntity.relativePath} with id : $localID', + ); + hasLocation = true; + } + } catch (e, s) { + _logger.severe('failed to get asset entity with id $localID', e, s); + } + if (hasLocation) { + localIDsWithLocation.add(localID); + } + } + _logger.info('marking ${localIDsWithLocation.length} files for re-upload'); + await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation); + await _fileUpdationDB.deleteByLocalIDs( + localIDsToProcess, + FileUpdationDB.missingLocation, + ); + } + + Future _importLocalFilesForMigration() async { + if (_prefs.containsKey(isLocalImportDone)) { + return; + } + final sTime = DateTime.now().microsecondsSinceEpoch; + _logger.info('importing files without location info'); + var fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation(); + await _fileUpdationDB.insertMultiple( + fileLocalIDs, + FileUpdationDB.missingLocation, + ); + final eTime = DateTime.now().microsecondsSinceEpoch; + final d = Duration(microseconds: eTime - sTime); + _logger.info( + 'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds', + ); + await _prefs.setBool(isLocalImportDone, true); + } +} diff --git a/lib/services/local_sync_service.dart b/lib/services/local_sync_service.dart index 9960635c6..60af14031 100644 --- a/lib/services/local_sync_service.dart +++ b/lib/services/local_sync_service.dart @@ -6,6 +6,7 @@ import 'package:logging/logging.dart'; import 'package:photo_manager/photo_manager.dart'; import 'package:photos/core/configuration.dart'; import 'package:photos/core/event_bus.dart'; +import 'package:photos/db/file_updation_db.dart'; import 'package:photos/db/files_db.dart'; import 'package:photos/events/local_photos_updated_event.dart'; import 'package:photos/events/sync_status_update_event.dart'; @@ -31,6 +32,7 @@ class LocalSyncService { // Adding `_2` as a suffic to pull files that were earlier ignored due to permission errors // See https://github.com/CaiJingLong/flutter_photo_manager/issues/589 static const kInvalidFileIDsKey = "invalid_file_ids_2"; + LocalSyncService._privateConstructor(); static final LocalSyncService instance = @@ -243,17 +245,15 @@ class LocalSyncService { updatedFiles.length.toString() + " local files were updated.", ); } + + final List updatedLocalIDs = []; for (final file in updatedFiles) { - await captureUpdateLogs(file); - await _db.updateUploadedFile( - file.localID, - file.title, - file.location, - file.creationTime, - file.modificationTime, - null, - ); + updatedLocalIDs.add(file.localID); } + await FileUpdationDB.instance.insertMultiple( + updatedLocalIDs, + FileUpdationDB.modificationTimeUpdated, + ); final List allFiles = []; allFiles.addAll(files); files.removeWhere((file) => existingLocalFileIDs.contains(file.localID)); @@ -265,32 +265,6 @@ class LocalSyncService { await _prefs.setInt(kDbUpdationTimeKey, toTime); } - // _captureUpdateLogs is a helper method to log details - // about the file which is being marked for re-upload - Future captureUpdateLogs(File file) async { - _logger.info( - 're-upload locally updated file ${file.toString()}', - ); - try { - if (Platform.isIOS) { - final assetEntity = await AssetEntity.fromId(file.localID); - if (assetEntity != null) { - final isLocallyAvailable = - await assetEntity.isLocallyAvailable(isOrigin: true); - _logger.info( - 're-upload asset ${file.toString()} with localAvailableFlag ' - '$isLocallyAvailable and fav ${assetEntity.isFavorite}', - ); - } else { - _logger - .info('re-upload failed to fetch assetInfo ${file.toString()}'); - } - } - } catch (ignore) { - //ignore - } - } - void _updatePathsToBackup(List files) { if (Configuration.instance.hasSelectedAllFoldersForBackup()) { final pathsToBackup = Configuration.instance.getPathsToBackUp(); diff --git a/lib/services/remote_sync_service.dart b/lib/services/remote_sync_service.dart index 23fad5dc6..d960d05c5 100644 --- a/lib/services/remote_sync_service.dart +++ b/lib/services/remote_sync_service.dart @@ -16,9 +16,8 @@ import 'package:photos/models/file.dart'; import 'package:photos/models/file_type.dart'; import 'package:photos/services/app_lifecycle_service.dart'; import 'package:photos/services/collections_service.dart'; -import 'package:photos/services/feature_flag_service.dart'; -import 'package:photos/services/file_migration_service.dart'; import 'package:photos/services/ignored_files_service.dart'; +import 'package:photos/services/local_file_update_service.dart'; import 'package:photos/services/local_sync_service.dart'; import 'package:photos/services/trash_sync_service.dart'; import 'package:photos/utils/diff_fetcher.dart'; @@ -32,8 +31,8 @@ class RemoteSyncService { final _uploader = FileUploader.instance; final _collectionsService = CollectionsService.instance; final _diffFetcher = DiffFetcher(); - final FileMigrationService _fileMigrationService = - FileMigrationService.instance; + final LocalFileUpdateService _localFileUpdateService = + LocalFileUpdateService.instance; int _completedUploads = 0; SharedPreferences _prefs; Completer _existingSync; @@ -134,10 +133,8 @@ class RemoteSyncService { if (!_hasReSynced()) { await _markReSyncAsDone(); } - if (FeatureFlagService.instance.enableMissingLocationMigration() && - !_fileMigrationService.isLocationMigrationCompleted()) { - _fileMigrationService.runMigration(); - } + + unawaited(_localFileUpdateService.markUpdatedFilesForReUpload()); } Future _syncUpdatedCollections(bool silently) async { diff --git a/lib/utils/file_uploader.dart b/lib/utils/file_uploader.dart index 4460ee53a..c6142b8f9 100644 --- a/lib/utils/file_uploader.dart +++ b/lib/utils/file_uploader.dart @@ -7,6 +7,7 @@ import 'dart:typed_data'; import 'package:connectivity/connectivity.dart'; import 'package:dio/dio.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter_sodium/flutter_sodium.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart'; @@ -31,6 +32,7 @@ import 'package:photos/utils/crypto_util.dart'; import 'package:photos/utils/file_download_util.dart'; import 'package:photos/utils/file_uploader_util.dart'; import 'package:shared_preferences/shared_preferences.dart'; +import 'package:tuple/tuple.dart'; class FileUploader { static const kMaximumConcurrentUploads = 4; @@ -108,6 +110,8 @@ class FileUploader { }); } + // upload future will return null as File when the file entry is deleted + // locally because it's already present in the destination collection. Future upload(File file, int collectionID) { // If the file hasn't been queued yet, queue it _totalCountInUploadSession++; @@ -125,10 +129,25 @@ class FileUploader { _totalCountInUploadSession--; return item.completer.future; } - + debugPrint( + "Wait on another upload on same local ID to finish before " + "adding it to new collection", + ); // Else wait for the existing upload to complete, // and add it to the relevant collection return item.completer.future.then((uploadedFile) { + // If the fileUploader completer returned null, + _logger.info( + "original upload completer resolved, try adding the file to another " + "collection", + ); + if (uploadedFile == null) { + /* todo: handle this case, ideally during next sync the localId + should be uploaded to this collection ID + */ + _logger.severe('unexpected upload state'); + return null; + } return CollectionsService.instance .addToCollection(collectionID, [uploadedFile]).then((aVoid) { return uploadedFile; @@ -136,61 +155,6 @@ class FileUploader { }); } - Future forceUpload(File file, int collectionID) async { - _logger.info( - "Force uploading " + - file.toString() + - " into collection " + - collectionID.toString(), - ); - _totalCountInUploadSession++; - // If the file hasn't been queued yet, ez. - if (!_queue.containsKey(file.localID)) { - final completer = Completer(); - _queue[file.localID] = FileUploadItem( - file, - collectionID, - completer, - status: UploadStatus.inProgress, - ); - _encryptAndUploadFileToCollection(file, collectionID, forcedUpload: true); - return completer.future; - } - var item = _queue[file.localID]; - // If the file is being uploaded right now, wait and proceed - if (item.status == UploadStatus.inProgress || - item.status == UploadStatus.inBackground) { - _totalCountInUploadSession--; - final uploadedFile = await item.completer.future; - if (uploadedFile.collectionID == collectionID) { - // Do nothing - } else { - await CollectionsService.instance - .addToCollection(collectionID, [uploadedFile]); - } - return uploadedFile; - } else { - // If the file is yet to be processed, - // 1. Set the status to in_progress - // 2. Force upload the file - // 3. Add to the relevant collection - item = _queue[file.localID]; - item.status = UploadStatus.inProgress; - final uploadedFile = await _encryptAndUploadFileToCollection( - file, - collectionID, - forcedUpload: true, - ); - if (item.collectionID == collectionID) { - return uploadedFile; - } else { - await CollectionsService.instance - .addToCollection(item.collectionID, [uploadedFile]); - return uploadedFile; - } - } - } - int getCurrentSessionUploadCount() { return _totalCountInUploadSession; } @@ -319,6 +283,7 @@ class FileUploader { fileOnDisk.updationTime != -1 && fileOnDisk.collectionID == collectionID; if (wasAlreadyUploaded) { + debugPrint("File is already uploaded ${fileOnDisk.tag()}"); return fileOnDisk; } @@ -362,6 +327,7 @@ class FileUploader { rethrow; } } + Uint8List key; final bool isUpdatedFile = file.uploadedFileID != null && file.updationTime == -1; @@ -370,6 +336,22 @@ class FileUploader { key = decryptFileKey(file); } else { key = null; + // check if the file is already uploaded and can be mapped to existing + // uploaded file. If map is found, it also returns the corresponding + // mapped or update file entry. + final result = await _mapToExistingUploadWithSameHash( + mediaUploadData, + file, + collectionID, + ); + final isMappedToExistingUpload = result.item1; + if (isMappedToExistingUpload) { + debugPrint( + "File success mapped to existing uploaded ${file.toString()}", + ); + // return the mapped file + return result.item2; + } } if (io.File(encryptedFilePath).existsSync()) { @@ -399,8 +381,7 @@ class FileUploader { final fileUploadURL = await _getUploadURL(); final String fileObjectKey = await _putFile(fileUploadURL, encryptedFile); - final metadata = - await file.getMetadataForUpload(mediaUploadData.sourceFile); + final metadata = await file.getMetadataForUpload(mediaUploadData); final encryptedMetadataData = await CryptoUtil.encryptChaCha( utf8.encode(jsonEncode(metadata)), fileAttributes.key, @@ -476,25 +457,143 @@ class FileUploader { } rethrow; } finally { - if (mediaUploadData != null && mediaUploadData.sourceFile != null) { - // delete the file from app's internal cache if it was copied to app - // for upload. Shared Media should only be cleared when the upload - // succeeds. - if (io.Platform.isIOS || - (uploadCompleted && file.isSharedMediaToAppSandbox())) { - await mediaUploadData.sourceFile.delete(); - } - } - if (io.File(encryptedFilePath).existsSync()) { - await io.File(encryptedFilePath).delete(); - } - if (io.File(encryptedThumbnailPath).existsSync()) { - await io.File(encryptedThumbnailPath).delete(); - } - await _uploadLocks.releaseLock(file.localID, _processType.toString()); + await _onUploadDone( + mediaUploadData, + uploadCompleted, + file, + encryptedFilePath, + encryptedThumbnailPath, + ); } } + /* + _mapToExistingUpload links the fileToUpload with the existing uploaded + files. if the link is successful, it returns true otherwise false. + When false, we should go ahead and re-upload or update the file. + It performs following checks: + a) Uploaded file with same localID and destination collection. Delete the + fileToUpload entry + b) Uploaded file in destination collection but with missing localID. + Update the localID for uploadedFile and delete the fileToUpload entry + c) A uploaded file exist with same localID but in a different collection. + or + d) Uploaded file in different collection but missing localID. + For both c and d, perform add to collection operation. + e) File already exists but different localID. Re-upload + In case the existing files already have local identifier, which is + different from the {fileToUpload}, then most probably device has + duplicate files. + */ + Future> _mapToExistingUploadWithSameHash( + MediaUploadData mediaUploadData, + File fileToUpload, + int toCollectionID, + ) async { + if (fileToUpload.uploadedFileID != null) { + // ideally this should never happen, but because the code below this case + // can do unexpected mapping, we are adding this additional check + _logger.severe( + 'Critical: file is already uploaded, skipped mapping', + ); + return Tuple2(false, fileToUpload); + } + + final List existingUploadedFiles = + await FilesDB.instance.getUploadedFilesWithHashes( + mediaUploadData.hashData, + fileToUpload.fileType, + Configuration.instance.getUserID(), + ); + if (existingUploadedFiles?.isEmpty ?? true) { + // continueUploading this file + return Tuple2(false, fileToUpload); + } else { + debugPrint("Found some matches"); + } + // case a + final File sameLocalSameCollection = existingUploadedFiles.firstWhere( + (e) => + e.collectionID == toCollectionID && e.localID == fileToUpload.localID, + orElse: () => null, + ); + if (sameLocalSameCollection != null) { + debugPrint( + "sameLocalSameCollection: \n toUpload ${fileToUpload.tag()} " + "\n existing: ${sameLocalSameCollection.tag()}", + ); + // should delete the fileToUploadEntry + await FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID); + return Tuple2(true, sameLocalSameCollection); + } + + // case b + final File fileMissingLocalButSameCollection = + existingUploadedFiles.firstWhere( + (e) => e.collectionID == toCollectionID && e.localID == null, + orElse: () => null, + ); + if (fileMissingLocalButSameCollection != null) { + // update the local id of the existing file and delete the fileToUpload + // entry + debugPrint( + "fileMissingLocalButSameCollection: \n toUpload ${fileToUpload.tag()} " + "\n existing: ${fileMissingLocalButSameCollection.tag()}", + ); + fileMissingLocalButSameCollection.localID = fileToUpload.localID; + await FilesDB.instance.insert(fileMissingLocalButSameCollection); + await FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID); + return Tuple2(true, fileMissingLocalButSameCollection); + } + + // case c and d + final File fileExistsButDifferentCollection = + existingUploadedFiles.firstWhere( + (e) => e.collectionID != toCollectionID, + orElse: () => null, + ); + if (fileExistsButDifferentCollection != null) { + debugPrint( + "fileExistsButDifferentCollection: \n toUpload ${fileToUpload.tag()} " + "\n existing: ${fileExistsButDifferentCollection.tag()}", + ); + final linkedFile = await CollectionsService.instance + .linkLocalFileToExistingUploadedFileInAnotherCollection( + toCollectionID, + localFileToUpload: fileToUpload, + existingUploadedFile: fileExistsButDifferentCollection, + ); + return Tuple2(true, linkedFile); + } + // case e + return Tuple2(false, fileToUpload); + } + + Future _onUploadDone( + MediaUploadData mediaUploadData, + bool uploadCompleted, + File file, + String encryptedFilePath, + String encryptedThumbnailPath, + ) async { + if (mediaUploadData != null && mediaUploadData.sourceFile != null) { + // delete the file from app's internal cache if it was copied to app + // for upload. Shared Media should only be cleared when the upload + // succeeds. + if (io.Platform.isIOS || + (uploadCompleted && file.isSharedMediaToAppSandbox())) { + await mediaUploadData.sourceFile.delete(); + } + } + if (io.File(encryptedFilePath).existsSync()) { + await io.File(encryptedFilePath).delete(); + } + if (io.File(encryptedThumbnailPath).existsSync()) { + await io.File(encryptedThumbnailPath).delete(); + } + await _uploadLocks.releaseLock(file.localID, _processType.toString()); + } + Future _onInvalidFileError(File file, InvalidFileError e) async { final String ext = file.title == null ? "no title" : extension(file.title); _logger.severe( diff --git a/lib/utils/file_uploader_util.dart b/lib/utils/file_uploader_util.dart index afea0e360..60bc10c69 100644 --- a/lib/utils/file_uploader_util.dart +++ b/lib/utils/file_uploader_util.dart @@ -3,6 +3,7 @@ import 'dart:io' as io; import 'dart:typed_data'; import 'package:archive/archive_io.dart'; +import 'package:flutter_sodium/flutter_sodium.dart'; import 'package:logging/logging.dart'; import 'package:motionphoto/motionphoto.dart'; import 'package:path/path.dart'; @@ -14,18 +15,37 @@ import 'package:photos/core/errors.dart'; import 'package:photos/models/file.dart' as ente; import 'package:photos/models/file_type.dart'; import 'package:photos/models/location.dart'; +import 'package:photos/utils/crypto_util.dart'; import 'package:photos/utils/file_util.dart'; import 'package:video_thumbnail/video_thumbnail.dart'; final _logger = Logger("FileUtil"); const kMaximumThumbnailCompressionAttempts = 2; +const kLivePhotoHashSeparator = ':'; class MediaUploadData { final io.File sourceFile; final Uint8List thumbnail; final bool isDeleted; + final FileHashData hashData; - MediaUploadData(this.sourceFile, this.thumbnail, this.isDeleted); + MediaUploadData( + this.sourceFile, + this.thumbnail, + this.isDeleted, + this.hashData, + ); +} + +class FileHashData { + // For livePhotos, the fileHash value will be imageHash:videoHash + final String fileHash; + + // zipHash is used to take care of existing live photo uploads from older + // mobile clients + String zipHash; + + FileHashData(this.fileHash, {this.zipHash}); } Future getUploadDataFromEnteFile(ente.File file) async { @@ -40,6 +60,7 @@ Future _getMediaUploadDataFromAssetFile(ente.File file) async { io.File sourceFile; Uint8List thumbnailData; bool isDeleted; + String fileHash, zipHash; // The timeouts are to safeguard against https://github.com/CaiJingLong/flutter_photo_manager/issues/467 final asset = await file @@ -72,6 +93,7 @@ Future _getMediaUploadDataFromAssetFile(ente.File file) async { // h4ck to fetch location data if missing (thank you Android Q+) lazily only during uploads await _decorateEnteFileData(file, asset); + fileHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile)); if (file.fileType == FileType.livePhoto && io.Platform.isIOS) { final io.File videoUrl = await Motionphoto.getLivePhotoFile(file.localID); @@ -81,6 +103,10 @@ Future _getMediaUploadDataFromAssetFile(ente.File file) async { _logger.severe(errMsg); throw InvalidFileUploadState(errMsg); } + String livePhotoVideoHash = + Sodium.bin2base64(await CryptoUtil.getHash(videoUrl)); + // imgHash:vidHash + fileHash = '$fileHash$kLivePhotoHashSeparator$livePhotoVideoHash'; final tempPath = Configuration.instance.getTempDirectory(); // .elp -> ente live photo final livePhotoPath = tempPath + file.generatedID.toString() + ".elp"; @@ -96,6 +122,7 @@ Future _getMediaUploadDataFromAssetFile(ente.File file) async { } // new sourceFile which needs to be uploaded sourceFile = io.File(livePhotoPath); + zipHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile)); } thumbnailData = await asset.thumbnailDataWithSize( @@ -116,7 +143,12 @@ Future _getMediaUploadDataFromAssetFile(ente.File file) async { } isDeleted = asset == null || !(await asset.exists); - return MediaUploadData(sourceFile, thumbnailData, isDeleted); + return MediaUploadData( + sourceFile, + thumbnailData, + isDeleted, + FileHashData(fileHash, zipHash: zipHash), + ); } Future _decorateEnteFileData(ente.File file, AssetEntity asset) async { @@ -128,7 +160,7 @@ Future _decorateEnteFileData(ente.File file, AssetEntity asset) async { } if (file.title == null || file.title.isEmpty) { - _logger.severe("Title was missing"); + _logger.warning("Title was missing ${file.tag()}"); file.title = await asset.titleAsync; } } @@ -145,7 +177,13 @@ Future _getMediaUploadDataFromAppCache(ente.File file) async { } try { thumbnailData = await getThumbnailFromInAppCacheFile(file); - return MediaUploadData(sourceFile, thumbnailData, isDeleted); + final fileHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile)); + return MediaUploadData( + sourceFile, + thumbnailData, + isDeleted, + FileHashData(fileHash), + ); } catch (e, s) { _logger.severe("failed to generate thumbnail", e, s); throw InvalidFileError(