|
@@ -7,8 +7,12 @@ import "package:photos/core/configuration.dart";
|
|
|
import 'package:photos/core/errors.dart';
|
|
|
import 'package:photos/db/file_updation_db.dart';
|
|
|
import 'package:photos/db/files_db.dart';
|
|
|
+import "package:photos/extensions/list.dart";
|
|
|
+import "package:photos/extensions/stop_watch.dart";
|
|
|
+import "package:photos/models/file/extensions/file_props.dart";
|
|
|
import 'package:photos/models/file/file.dart';
|
|
|
import 'package:photos/models/file/file_type.dart';
|
|
|
+import "package:photos/services/files_service.dart";
|
|
|
import 'package:photos/utils/file_uploader_util.dart';
|
|
|
import 'package:photos/utils/file_util.dart';
|
|
|
import 'package:shared_preferences/shared_preferences.dart';
|
|
@@ -19,6 +23,9 @@ class LocalFileUpdateService {
|
|
|
late FileUpdationDB _fileUpdationDB;
|
|
|
late SharedPreferences _prefs;
|
|
|
late Logger _logger;
|
|
|
+ final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_check';
|
|
|
+ final String _doneLivePhotoImport = 'fm_import_ios_live_photo_check';
|
|
|
+ static int twoHundredKb = 200 * 1024;
|
|
|
final List<String> _oldMigrationKeys = [
|
|
|
'fm_badCreationTime',
|
|
|
'fm_badCreationTimeCompleted',
|
|
@@ -26,6 +33,8 @@ class LocalFileUpdateService {
|
|
|
'fm_missingLocationV2MigrationDone',
|
|
|
'fm_badLocationImportDone',
|
|
|
'fm_badLocationMigrationDone',
|
|
|
+ 'fm_ios_live_photo_size',
|
|
|
+ 'fm_import_ios_live_photo_size',
|
|
|
];
|
|
|
|
|
|
Completer<void>? _existingMigration;
|
|
@@ -50,8 +59,9 @@ class LocalFileUpdateService {
|
|
|
_existingMigration = Completer<void>();
|
|
|
try {
|
|
|
await _markFilesWhichAreActuallyUpdated();
|
|
|
- if (Platform.isAndroid) {
|
|
|
- _cleanUpOlderMigration().ignore();
|
|
|
+ _cleanUpOlderMigration().ignore();
|
|
|
+ if (!Platform.isAndroid) {
|
|
|
+ await _handleLivePhotosSizedCheck();
|
|
|
}
|
|
|
} catch (e, s) {
|
|
|
_logger.severe('failed to perform migration', e, s);
|
|
@@ -71,15 +81,16 @@ class LocalFileUpdateService {
|
|
|
}
|
|
|
}
|
|
|
if (hasOldMigrationKey) {
|
|
|
- for (var element in _oldMigrationKeys) {
|
|
|
- _prefs.remove(element);
|
|
|
- }
|
|
|
await _fileUpdationDB.deleteByReasons([
|
|
|
'missing_location',
|
|
|
'badCreationTime',
|
|
|
'missingLocationV2',
|
|
|
'badLocationCord',
|
|
|
+ 'livePhotoSize',
|
|
|
]);
|
|
|
+ for (var element in _oldMigrationKeys) {
|
|
|
+ await _prefs.remove(element);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -199,6 +210,181 @@ class LocalFileUpdateService {
|
|
|
);
|
|
|
}
|
|
|
|
|
|
+ Future<void> checkLivePhoto(EnteFile file) async {
|
|
|
+ if (file.localID == null ||
|
|
|
+ file.localID!.isEmpty ||
|
|
|
+ !file.isUploaded ||
|
|
|
+ file.fileType != FileType.livePhoto ||
|
|
|
+ !file.isOwner) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ bool hasEntry = await _fileUpdationDB.isExisting(
|
|
|
+ file.localID!,
|
|
|
+ FileUpdationDB.livePhotoCheck,
|
|
|
+ );
|
|
|
+ if (hasEntry) {
|
|
|
+ _logger.info('eager checkLivePhoto ${file.tag}');
|
|
|
+ await _checkLivePhotoWithLowOrUnknownSize([file.localID!]);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ Future<void> _handleLivePhotosSizedCheck() async {
|
|
|
+ try {
|
|
|
+ if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ await _importLivePhotoReUploadCandidates();
|
|
|
+
|
|
|
+ // singleRunLimit indicates number of files to check during single
|
|
|
+ // invocation of this method. The limit act as a crude way to limit the
|
|
|
+ // resource consumed by the method
|
|
|
+ const int singleRunLimit = 500;
|
|
|
+ final localIDsToProcess =
|
|
|
+ await _fileUpdationDB.getLocalIDsForPotentialReUpload(
|
|
|
+ singleRunLimit,
|
|
|
+ FileUpdationDB.livePhotoCheck,
|
|
|
+ );
|
|
|
+ if (localIDsToProcess.isNotEmpty) {
|
|
|
+ final chunksOf50 = localIDsToProcess.chunks(50);
|
|
|
+ for (final chunk in chunksOf50) {
|
|
|
+ final sTime = DateTime.now().microsecondsSinceEpoch;
|
|
|
+ final List<Future> futures = [];
|
|
|
+ final chunkOf10 = chunk.chunks(10);
|
|
|
+ for (final smallChunk in chunkOf10) {
|
|
|
+ futures.add(_checkLivePhotoWithLowOrUnknownSize(smallChunk));
|
|
|
+ }
|
|
|
+ await Future.wait(futures);
|
|
|
+ final eTime = DateTime.now().microsecondsSinceEpoch;
|
|
|
+ final d = Duration(microseconds: eTime - sTime);
|
|
|
+ _logger.info(
|
|
|
+ 'Performed hashCheck for ${chunk.length} livePhoto files '
|
|
|
+ 'completed in ${d.inSeconds.toString()} secs',
|
|
|
+ );
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ await _prefs.setBool(_iosLivePhotoSizeMigrationDone, true);
|
|
|
+ }
|
|
|
+ } catch (e, s) {
|
|
|
+ _logger.severe('error while checking livePhotoSize check', e, s);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ Future<void> _checkLivePhotoWithLowOrUnknownSize(
|
|
|
+ List<String> localIDsToProcess,
|
|
|
+ ) async {
|
|
|
+ final int userID = Configuration.instance.getUserID()!;
|
|
|
+ final List<EnteFile> result =
|
|
|
+ await FilesDB.instance.getLocalFiles(localIDsToProcess);
|
|
|
+ final List<EnteFile> localFilesForUser = [];
|
|
|
+ final Set<String> localIDsWithFile = {};
|
|
|
+ final Set<int> missingSizeIDs = {};
|
|
|
+ final Set<String> processedIDs = {};
|
|
|
+ for (EnteFile file in result) {
|
|
|
+ if (file.ownerID == null || file.ownerID == userID) {
|
|
|
+ localFilesForUser.add(file);
|
|
|
+ localIDsWithFile.add(file.localID!);
|
|
|
+ if (file.isUploaded && file.fileSize == null) {
|
|
|
+ missingSizeIDs.add(file.uploadedFileID!);
|
|
|
+ }
|
|
|
+ if (file.isUploaded && file.updationTime == null) {
|
|
|
+ // file already queued for re-upload
|
|
|
+ processedIDs.add(file.localID!);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (missingSizeIDs.isNotEmpty) {
|
|
|
+ await FilesService.instance.backFillSizes(missingSizeIDs.toList());
|
|
|
+ _logger.info('sizes back fill for ${missingSizeIDs.length} files');
|
|
|
+ // return early, let the check run in the next batch
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // if a file for localID doesn't exist, then mark it as processed
|
|
|
+ // otherwise the app will be stuck in retrying same set of ids
|
|
|
+
|
|
|
+ for (String localID in localIDsToProcess) {
|
|
|
+ if (!localIDsWithFile.contains(localID)) {
|
|
|
+ processedIDs.add(localID);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ _logger.info(" check ${localIDsToProcess.length} files for livePhotoSize, "
|
|
|
+ "missing file cnt ${processedIDs.length}");
|
|
|
+
|
|
|
+ for (EnteFile file in localFilesForUser) {
|
|
|
+ if (file.fileSize == null) {
|
|
|
+ _logger.info('fileSize still null, skip this file');
|
|
|
+ continue;
|
|
|
+ } else if (file.fileType != FileType.livePhoto) {
|
|
|
+ _logger.severe('fileType is not livePhoto, skip this file');
|
|
|
+ processedIDs.add(file.localID!);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (processedIDs.contains(file.localID)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ late MediaUploadData uploadData;
|
|
|
+ late int mediaUploadSize;
|
|
|
+ (uploadData, mediaUploadSize) = await getUploadDataWithSizeSize(file);
|
|
|
+ if ((file.fileSize! - mediaUploadSize).abs() > twoHundredKb) {
|
|
|
+ _logger.info(
|
|
|
+ 'Re-upload livePhoto localHash ${uploadData.hashData?.fileHash ?? "null"} & localSize: $mediaUploadSize'
|
|
|
+ ' and remoteHash ${file.hash ?? "null"} & removeSize: ${file.fileSize!}',
|
|
|
+ );
|
|
|
+ await FilesDB.instance.markFilesForReUpload(
|
|
|
+ userID,
|
|
|
+ file.localID!,
|
|
|
+ file.title,
|
|
|
+ file.location,
|
|
|
+ file.creationTime!,
|
|
|
+ file.modificationTime!,
|
|
|
+ file.fileType,
|
|
|
+ );
|
|
|
+ }
|
|
|
+ processedIDs.add(file.localID!);
|
|
|
+ } on InvalidFileError catch (e) {
|
|
|
+ if (e.reason == InvalidReason.livePhotoToImageTypeChanged ||
|
|
|
+ e.reason == InvalidReason.imageToLivePhotoTypeChanged) {
|
|
|
+ // let existing file update check handle this case
|
|
|
+ _fileUpdationDB.insertMultiple(
|
|
|
+ [file.localID!],
|
|
|
+ FileUpdationDB.modificationTimeUpdated,
|
|
|
+ ).ignore();
|
|
|
+ } else {
|
|
|
+ _logger.severe("livePhoto check failed: invalid file ${file.tag}", e);
|
|
|
+ }
|
|
|
+ processedIDs.add(file.localID!);
|
|
|
+ } catch (e) {
|
|
|
+ _logger.severe("livePhoto check failed", e);
|
|
|
+ } finally {}
|
|
|
+ }
|
|
|
+ _logger.info('completed check for ${localIDsToProcess.length} files');
|
|
|
+ await _fileUpdationDB.deleteByLocalIDs(
|
|
|
+ processedIDs.toList(),
|
|
|
+ FileUpdationDB.livePhotoCheck,
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ Future<void> _importLivePhotoReUploadCandidates() async {
|
|
|
+ if (_prefs.containsKey(_doneLivePhotoImport)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ _logger.info('_importLivePhotoReUploadCandidates');
|
|
|
+ final EnteWatch watch = EnteWatch("_importLivePhotoReUploadCandidates");
|
|
|
+ final int ownerID = Configuration.instance.getUserID()!;
|
|
|
+ final List<String> localIDs =
|
|
|
+ await FilesDB.instance.getLivePhotosForUser(ownerID);
|
|
|
+ await _fileUpdationDB.insertMultiple(
|
|
|
+ localIDs,
|
|
|
+ FileUpdationDB.livePhotoCheck,
|
|
|
+ );
|
|
|
+ watch.log("imported ${localIDs.length} files");
|
|
|
+ await _prefs.setBool(_doneLivePhotoImport, true);
|
|
|
+ }
|
|
|
+
|
|
|
Future<MediaUploadData> getUploadData(EnteFile file) async {
|
|
|
final mediaUploadData = await getUploadDataFromEnteFile(file);
|
|
|
// delete the file from app's internal cache if it was copied to app
|
|
@@ -209,4 +395,18 @@ class LocalFileUpdateService {
|
|
|
}
|
|
|
return mediaUploadData;
|
|
|
}
|
|
|
+
|
|
|
+ Future<(MediaUploadData, int)> getUploadDataWithSizeSize(
|
|
|
+ EnteFile file,
|
|
|
+ ) async {
|
|
|
+ final mediaUploadData = await getUploadDataFromEnteFile(file);
|
|
|
+ final int size = await mediaUploadData.sourceFile!.length();
|
|
|
+ // delete the file from app's internal cache if it was copied to app
|
|
|
+ // for upload. Shared Media should only be cleared when the upload
|
|
|
+ // succeeds.
|
|
|
+ if (Platform.isIOS && mediaUploadData.sourceFile != null) {
|
|
|
+ await mediaUploadData.sourceFile?.delete();
|
|
|
+ }
|
|
|
+ return (mediaUploadData, size);
|
|
|
+ }
|
|
|
}
|