diff --git a/lib/db/file_updation_db.dart b/lib/db/file_updation_db.dart index 22f317924..656f8ed17 100644 --- a/lib/db/file_updation_db.dart +++ b/lib/db/file_updation_db.dart @@ -14,6 +14,7 @@ class FileUpdationDB { static const tableName = 're_upload_tracker'; static const columnLocalID = 'local_id'; static const columnReason = 'reason'; + static const livePhotoSize = 'livePhotoSize'; static const modificationTimeUpdated = 'modificationTimeUpdated'; diff --git a/lib/db/files_db.dart b/lib/db/files_db.dart index 4864130b2..0bb1f99e3 100644 --- a/lib/db/files_db.dart +++ b/lib/db/files_db.dart @@ -1449,6 +1449,28 @@ class FilesDB { return result; } + // For a given userID, return unique uploadedFileId for the given userID + Future> getLivePhotosWithBadSize( + int userId, + int sizeInBytes, + ) async { + final db = await instance.database; + final rows = await db.query( + filesTable, + columns: [columnLocalID], + distinct: true, + where: '$columnOwnerID = ? AND ' + '($columnFileSize IS NULL OR $columnFileSize = ?) AND ' + '$columnFileType = ? AND $columnLocalID IS NOT NULL', + whereArgs: [userId, sizeInBytes, getInt(FileType.livePhoto)], + ); + final result = []; + for (final row in rows) { + result.add(row[columnLocalID] as String); + } + return result; + } + // updateSizeForUploadIDs takes a map of upploadedFileID and fileSize and // update the fileSize for the given uploadedFileID Future updateSizeForUploadIDs( diff --git a/lib/services/files_service.dart b/lib/services/files_service.dart index 8d50fa9f2..d97b8a25e 100644 --- a/lib/services/files_service.dart +++ b/lib/services/files_service.dart @@ -49,11 +49,7 @@ class FilesService { if (uploadIDsWithMissingSize.isEmpty) { return Future.value(true); } - final batchedFiles = uploadIDsWithMissingSize.chunks(1000); - for (final batch in batchedFiles) { - final Map uploadIdToSize = await getFilesSizeFromInfo(batch); - await _filesDB.updateSizeForUploadIDs(uploadIdToSize); - } + await backFillSizes(uploadIDsWithMissingSize); return Future.value(true); } catch (e, s) { _logger.severe("error during has migrated sizes", e, s); @@ -61,6 +57,14 @@ class FilesService { } } + Future backFillSizes(List uploadIDsWithMissingSize) async { + final batchedFiles = uploadIDsWithMissingSize.chunks(1000); + for (final batch in batchedFiles) { + final Map uploadIdToSize = await getFilesSizeFromInfo(batch); + await _filesDB.updateSizeForUploadIDs(uploadIdToSize); + } + } + Future> getFilesSizeFromInfo(List uploadedFileID) async { try { final response = await _enteDio.post( diff --git a/lib/services/local_file_update_service.dart b/lib/services/local_file_update_service.dart index 52ba069fb..d77e0c12b 100644 --- a/lib/services/local_file_update_service.dart +++ b/lib/services/local_file_update_service.dart @@ -7,8 +7,10 @@ import "package:photos/core/configuration.dart"; import 'package:photos/core/errors.dart'; import 'package:photos/db/file_updation_db.dart'; import 'package:photos/db/files_db.dart'; +import "package:photos/extensions/stop_watch.dart"; import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; +import "package:photos/services/files_service.dart"; import 'package:photos/utils/file_uploader_util.dart'; import 'package:photos/utils/file_util.dart'; import 'package:shared_preferences/shared_preferences.dart'; @@ -19,6 +21,9 @@ class LocalFileUpdateService { late FileUpdationDB _fileUpdationDB; late SharedPreferences _prefs; late Logger _logger; + final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_size'; + final String _doneLivePhotoImport = 'fm_import_ios_live_photo_size'; + static int fourMBWithChunkSize = 4194338; final List _oldMigrationKeys = [ 'fm_badCreationTime', 'fm_badCreationTimeCompleted', @@ -52,6 +57,8 @@ class LocalFileUpdateService { await _markFilesWhichAreActuallyUpdated(); if (Platform.isAndroid) { _cleanUpOlderMigration().ignore(); + } else { + await _handleLivePhotosSizedCheck(); } } catch (e, s) { _logger.severe('failed to perform migration', e, s); @@ -199,6 +206,146 @@ class LocalFileUpdateService { ); } + Future _handleLivePhotosSizedCheck() async { + try { + if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) { + return; + } + await _importLivePhotoReUploadCandidates(); + final sTime = DateTime.now().microsecondsSinceEpoch; + // singleRunLimit indicates number of files to check during single + // invocation of this method. The limit act as a crude way to limit the + // resource consumed by the method + const int singleRunLimit = 50; + final localIDsToProcess = + await _fileUpdationDB.getLocalIDsForPotentialReUpload( + singleRunLimit, + FileUpdationDB.livePhotoSize, + ); + if (localIDsToProcess.isNotEmpty) { + await _checkLivePhotoWithLowOrUnknownSize( + localIDsToProcess, + ); + final eTime = DateTime.now().microsecondsSinceEpoch; + final d = Duration(microseconds: eTime - sTime); + _logger.info( + 'Performed hashCheck for ${localIDsToProcess.length} livePhoto files ' + 'completed in ${d.inSeconds.toString()} secs', + ); + } else { + _prefs.setBool(_iosLivePhotoSizeMigrationDone, true); + } + } catch (e, s) { + _logger.severe('error while checking livePhotoSize check', e, s); + } + } + + Future _checkLivePhotoWithLowOrUnknownSize( + List localIDsToProcess, + ) async { + final int userID = Configuration.instance.getUserID()!; + final List result = + await FilesDB.instance.getLocalFiles(localIDsToProcess); + final List localFilesForUser = []; + final Set localIDsWithFile = {}; + final Set missingSizeIDs = {}; + for (EnteFile file in result) { + if (file.ownerID == null || file.ownerID == userID) { + localFilesForUser.add(file); + localIDsWithFile.add(file.localID!); + if (file.isUploaded && file.fileSize == null) { + missingSizeIDs.add(file.uploadedFileID!); + } + } + } + if (missingSizeIDs.isNotEmpty) { + await FilesService.instance.backFillSizes(missingSizeIDs.toList()); + _logger.info('sizes back fill for ${missingSizeIDs.length} files'); + // return early, let the check run in the next batch + return; + } + + final Set processedIDs = {}; + // if a file for localID doesn't exist, then mark it as processed + // otherwise the app will be stuck in retrying same set of ids + + for (String localID in localIDsToProcess) { + if (!localIDsWithFile.contains(localID)) { + processedIDs.add(localID); + } + } + _logger.info(" check ${localIDsToProcess.length} files for livePhotoSize, " + "missing file cnt ${processedIDs.length}"); + + for (EnteFile file in localFilesForUser) { + if (file.fileSize == null) { + _logger.info('fileSize still null, skip this file'); + continue; + } else if (file.fileType != FileType.livePhoto) { + _logger.severe('fileType is not livePhoto, skip this file'); + continue; + } else if (file.fileSize! != fourMBWithChunkSize) { + // back-filled size is not of our interest + processedIDs.add(file.localID!); + continue; + } + if (processedIDs.contains(file.localID)) { + continue; + } + try { + final MediaUploadData uploadData = await getUploadData(file); + _logger.info( + 'Found livePhoto on local with hash ${uploadData.hashData?.fileHash ?? "null"} and existing hash ${file.hash ?? "null"}'); + await clearCache(file); + await FilesDB.instance.markFilesForReUpload( + userID, + file.localID!, + file.title, + file.location, + file.creationTime!, + file.modificationTime!, + file.fileType, + ); + processedIDs.add(file.localID!); + } on InvalidFileError catch (e) { + if (e.reason == InvalidReason.livePhotoToImageTypeChanged || + e.reason == InvalidReason.imageToLivePhotoTypeChanged) { + // let existing file update check handle this case + _fileUpdationDB.insertMultiple( + [file.localID!], + FileUpdationDB.modificationTimeUpdated, + ).ignore(); + } else { + _logger.severe("livePhoto check failed: invalid file ${file.tag}", e); + } + processedIDs.add(file.localID!); + } catch (e) { + _logger.severe("livePhoto check failed", e); + } finally {} + } + await _fileUpdationDB.deleteByLocalIDs( + processedIDs.toList(), + FileUpdationDB.livePhotoSize, + ); + } + + Future _importLivePhotoReUploadCandidates() async { + if (_prefs.containsKey(_doneLivePhotoImport)) { + return; + } + _logger.info('_importLivePhotoReUploadCandidates'); + final EnteWatch watch = EnteWatch("_importLivePhotoReUploadCandidates"); + final int ownerID = Configuration.instance.getUserID()!; + final List localIDs = await FilesDB.instance + .getLivePhotosWithBadSize(ownerID, fourMBWithChunkSize); + await _fileUpdationDB.insertMultiple( + localIDs, + FileUpdationDB.livePhotoSize, + ); + watch.log("imported ${localIDs.length} files"); + await _prefs.setBool(_doneLivePhotoImport, true); + } + Future getUploadData(EnteFile file) async { final mediaUploadData = await getUploadDataFromEnteFile(file); // delete the file from app's internal cache if it was copied to app diff --git a/pubspec.lock b/pubspec.lock index 4f7d290ea..cbe184e30 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -503,11 +503,12 @@ packages: file_saver: dependency: "direct main" description: - name: file_saver - sha256: "591d25e750e3a4b654f7b0293abc2ed857242f82ca7334051b2a8ceeb369dac8" - url: "https://pub.dev" - source: hosted - version: "0.2.8" + path: "." + ref: HEAD + resolved-ref: "01b2e6b6fe520cfa5d2d91342ccbfbaefa8f6482" + url: "https://github.com/jesims/file_saver.git" + source: git + version: "0.2.9" firebase_core: dependency: "direct main" description: diff --git a/pubspec.yaml b/pubspec.yaml index 40a7a4d7c..aa55370b4 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.9+529 +version: 0.8.10+530 environment: sdk: ">=3.0.0 <4.0.0" @@ -55,8 +55,10 @@ dependencies: extended_image: ^8.1.1 fade_indexed_stack: ^0.2.2 fast_base58: ^0.2.1 - # https://github.com/incrediblezayed/file_saver/issues/86 - file_saver: 0.2.8 + + file_saver: + # Use forked version till this PR is merged: https://github.com/incrediblezayed/file_saver/pull/87 + git: https://github.com/jesims/file_saver.git firebase_core: ^2.13.1 firebase_messaging: ^14.6.2 fk_user_agent: ^2.0.1