|
@@ -8,13 +8,14 @@ import 'package:photos/core/configuration.dart';
|
|
|
import 'package:photos/core/errors.dart';
|
|
|
import 'package:photos/core/event_bus.dart';
|
|
|
import 'package:photos/db/device_files_db.dart';
|
|
|
+import 'package:photos/db/file_updation_db.dart';
|
|
|
import 'package:photos/db/files_db.dart';
|
|
|
import 'package:photos/events/collection_updated_event.dart';
|
|
|
import 'package:photos/events/files_updated_event.dart';
|
|
|
import 'package:photos/events/force_reload_home_gallery_event.dart';
|
|
|
import 'package:photos/events/local_photos_updated_event.dart';
|
|
|
import 'package:photos/events/sync_status_update_event.dart';
|
|
|
-import 'package:photos/models/device_folder.dart';
|
|
|
+import 'package:photos/models/device_collection.dart';
|
|
|
import 'package:photos/models/file.dart';
|
|
|
import 'package:photos/models/file_type.dart';
|
|
|
import 'package:photos/services/app_lifecycle_service.dart';
|
|
@@ -39,6 +40,7 @@ class RemoteSyncService {
|
|
|
int _completedUploads = 0;
|
|
|
SharedPreferences _prefs;
|
|
|
Completer<void> _existingSync;
|
|
|
+ bool _existingSyncSilent = false;
|
|
|
|
|
|
static const kHasSyncedArchiveKey = "has_synced_archive";
|
|
|
|
|
@@ -75,12 +77,18 @@ class RemoteSyncService {
|
|
|
}
|
|
|
if (_existingSync != null) {
|
|
|
_logger.info("Remote sync already in progress, skipping");
|
|
|
+ // if current sync is silent but request sync is non-silent (demands UI
|
|
|
+ // updates), update the syncSilently flag
|
|
|
+ if (_existingSyncSilent == true && silently == false) {
|
|
|
+ _existingSyncSilent = false;
|
|
|
+ }
|
|
|
return _existingSync.future;
|
|
|
}
|
|
|
_existingSync = Completer<void>();
|
|
|
+ _existingSyncSilent = silently;
|
|
|
|
|
|
try {
|
|
|
- await _pullDiff(silently);
|
|
|
+ await _pullDiff();
|
|
|
// sync trash but consume error during initial launch.
|
|
|
// this is to ensure that we don't pause upload due to any error during
|
|
|
// the trash sync. Impact: We may end up re-uploading a file which was
|
|
@@ -96,7 +104,7 @@ class RemoteSyncService {
|
|
|
}
|
|
|
final hasUploadedFiles = await _uploadFiles(filesToBeUploaded);
|
|
|
if (hasUploadedFiles) {
|
|
|
- await _pullDiff(true);
|
|
|
+ await _pullDiff();
|
|
|
_existingSync.complete();
|
|
|
_existingSync = null;
|
|
|
final hasMoreFilesToBackup = (await _getFilesToBeUploaded()).isNotEmpty;
|
|
@@ -125,33 +133,31 @@ class RemoteSyncService {
|
|
|
} else {
|
|
|
_logger.severe("Error executing remote sync ", e, s);
|
|
|
}
|
|
|
+ } finally {
|
|
|
+ _existingSyncSilent = false;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- Future<void> _pullDiff(bool silently) async {
|
|
|
+ Future<void> _pullDiff() async {
|
|
|
final isFirstSync = !_collectionsService.hasSyncedCollections();
|
|
|
await _collectionsService.sync();
|
|
|
-
|
|
|
- if (isFirstSync || _hasReSynced()) {
|
|
|
- await _syncUpdatedCollections(silently);
|
|
|
- } else {
|
|
|
- final syncSinceTime = _getSinceTimeForReSync();
|
|
|
- await _resyncAllCollectionsSinceTime(syncSinceTime);
|
|
|
- }
|
|
|
- if (!_hasReSynced()) {
|
|
|
- await _markReSyncAsDone();
|
|
|
+ // check and reset user's collection syncTime in past for older clients
|
|
|
+ if (isFirstSync) {
|
|
|
+ // not need reset syncTime, mark all flags as done if firstSync
|
|
|
+ await _markResetSyncTimeAsDone();
|
|
|
+ } else if (_shouldResetSyncTime()) {
|
|
|
+ _logger.warning('Resetting syncTime for for the client');
|
|
|
+ await _resetAllCollectionsSyncTime();
|
|
|
+ await _markResetSyncTimeAsDone();
|
|
|
}
|
|
|
|
|
|
+ await _syncUpdatedCollections();
|
|
|
unawaited(_localFileUpdateService.markUpdatedFilesForReUpload());
|
|
|
}
|
|
|
|
|
|
- Future<void> _syncUpdatedCollections(bool silently) async {
|
|
|
+ Future<void> _syncUpdatedCollections() async {
|
|
|
final updatedCollections =
|
|
|
await _collectionsService.getCollectionsToBeSynced();
|
|
|
-
|
|
|
- if (updatedCollections.isNotEmpty && !silently) {
|
|
|
- Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff));
|
|
|
- }
|
|
|
for (final c in updatedCollections) {
|
|
|
await _syncCollectionDiff(
|
|
|
c.id,
|
|
@@ -161,19 +167,21 @@ class RemoteSyncService {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- Future<void> _resyncAllCollectionsSinceTime(int sinceTime) async {
|
|
|
- _logger.info('re-sync collections sinceTime: $sinceTime');
|
|
|
+ Future<void> _resetAllCollectionsSyncTime() async {
|
|
|
+ final resetSyncTime = _getSinceTimeForReSync();
|
|
|
+ _logger.info('re-setting all collections syncTime to: $resetSyncTime');
|
|
|
final collections = _collectionsService.getActiveCollections();
|
|
|
for (final c in collections) {
|
|
|
- await _syncCollectionDiff(
|
|
|
- c.id,
|
|
|
- min(_collectionsService.getCollectionSyncTime(c.id), sinceTime),
|
|
|
- );
|
|
|
- await _collectionsService.setCollectionSyncTime(c.id, c.updationTime);
|
|
|
+ final int newSyncTime =
|
|
|
+ min(_collectionsService.getCollectionSyncTime(c.id), resetSyncTime);
|
|
|
+ await _collectionsService.setCollectionSyncTime(c.id, newSyncTime);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
Future<void> _syncCollectionDiff(int collectionID, int sinceTime) async {
|
|
|
+ if (!_existingSyncSilent) {
|
|
|
+ Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff));
|
|
|
+ }
|
|
|
final diff =
|
|
|
await _diffFetcher.getEncryptedFilesDiff(collectionID, sinceTime);
|
|
|
if (diff.deletedFiles.isNotEmpty) {
|
|
@@ -456,133 +464,182 @@ class RemoteSyncService {
|
|
|
);
|
|
|
}
|
|
|
|
|
|
+ /* _storeDiff maps each remoteDiff file to existing
|
|
|
+ entries in files table. When match is found, it compares both file to
|
|
|
+ perform relevant actions like
|
|
|
+ [1] Clear local cache when required (Both Shared and Owned files)
|
|
|
+ [2] Retain localID of remote file based on matching logic [Owned files]
|
|
|
+ [3] Refresh UI if visibility or creationTime has changed [Owned files]
|
|
|
+ [4] Schedule file update if the local file has changed since last time
|
|
|
+ [Owned files]
|
|
|
+ [Important Note: If given uploadedFileID and collectionID is already present
|
|
|
+ in files db, the generateID should already point to existing entry.
|
|
|
+ Known Issues:
|
|
|
+ [K1] Cached entry will not be cleared when if a file was edited and
|
|
|
+ moved to different collection as Vid/Image cache key is uploadedID.
|
|
|
+ [Existing]
|
|
|
+ ]
|
|
|
+ */
|
|
|
Future _storeDiff(List<File> diff, int collectionID) async {
|
|
|
- int existing = 0,
|
|
|
- updated = 0,
|
|
|
- remote = 0,
|
|
|
- localButUpdatedOnRemote = 0,
|
|
|
- localButAddedToNewCollectionOnRemote = 0;
|
|
|
- bool hasAnyCreationTimeChanged = false;
|
|
|
- final List<File> toBeInserted = [];
|
|
|
+ int sharedFileNew = 0,
|
|
|
+ sharedFileUpdated = 0,
|
|
|
+ localUploadedFromDevice = 0,
|
|
|
+ localButUpdatedOnDevice = 0,
|
|
|
+ remoteNewFile = 0;
|
|
|
final int userID = Configuration.instance.getUserID();
|
|
|
- for (File file in diff) {
|
|
|
- final existingFiles = file.deviceFolder == null
|
|
|
- ? null
|
|
|
- : await _db.getMatchingFiles(file.title, file.deviceFolder);
|
|
|
- if (existingFiles == null ||
|
|
|
- existingFiles.isEmpty ||
|
|
|
- userID != file.ownerID) {
|
|
|
- // File uploaded from a different device or uploaded by different user
|
|
|
- // Other rare possibilities : The local file is present on
|
|
|
- // device but it's not imported in local db due to missing permission
|
|
|
- // after reinstall (iOS selected file permissions or user revoking
|
|
|
- // permissions, or issue/delay in importing devices files.
|
|
|
- file.localID = null;
|
|
|
- toBeInserted.add(file);
|
|
|
- remote++;
|
|
|
- } else {
|
|
|
- // File exists in ente db with same title & device folder
|
|
|
- // Note: The file.generatedID might be already set inside
|
|
|
- // [DiffFetcher.getEncryptedFilesDiff]
|
|
|
- // Try to find existing file with same localID as remote file with a fallback
|
|
|
- // to finding any existing file with localID. This is needed to handle
|
|
|
- // case when localID for a file changes and the file is uploaded again in
|
|
|
- // the same collection
|
|
|
- final fileWithLocalID = existingFiles.firstWhere(
|
|
|
- (e) =>
|
|
|
- file.localID != null &&
|
|
|
- e.localID != null &&
|
|
|
- e.localID == file.localID,
|
|
|
- orElse: () => existingFiles.firstWhere(
|
|
|
- (e) => e.localID != null,
|
|
|
- orElse: () => null,
|
|
|
- ),
|
|
|
- );
|
|
|
- if (fileWithLocalID != null) {
|
|
|
- // File should ideally have the same localID
|
|
|
- if (file.localID != null && file.localID != fileWithLocalID.localID) {
|
|
|
- _logger.severe(
|
|
|
- "unexpected mismatch in localIDs remote: ${file.toString()} and existing: ${fileWithLocalID.toString()}",
|
|
|
- );
|
|
|
- }
|
|
|
- file.localID = fileWithLocalID.localID;
|
|
|
+ bool needsGalleryReload = false;
|
|
|
+ // this is required when same file is uploaded twice in the same
|
|
|
+ // collection. Without this check, if both remote files are part of same
|
|
|
+ // diff response, then we end up inserting one entry instead of two
|
|
|
+ // as we update the generatedID for remoteDiff to local file's genID
|
|
|
+ final Set<int> alreadyClaimedLocalFilesGenID = {};
|
|
|
+
|
|
|
+ final List<File> toBeInserted = [];
|
|
|
+ for (File remoteDiff in diff) {
|
|
|
+ // existingFile will be either set to existing collectionID+localID or
|
|
|
+ // to the unclaimed aka not already linked to any uploaded file.
|
|
|
+ File existingFile;
|
|
|
+ if (remoteDiff.generatedID != null) {
|
|
|
+ // Case [1] Check and clear local cache when uploadedFile already exist
|
|
|
+ existingFile = await _db.getFile(remoteDiff.generatedID);
|
|
|
+ if (_shouldClearCache(remoteDiff, existingFile)) {
|
|
|
+ await clearCache(remoteDiff);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /* If file is not owned by the user, no further processing is required
|
|
|
+ as Case [2,3,4] are only relevant to files owned by user
|
|
|
+ */
|
|
|
+ if (userID != remoteDiff.ownerID) {
|
|
|
+ if (existingFile == null) {
|
|
|
+ sharedFileNew++;
|
|
|
+ remoteDiff.localID = null;
|
|
|
} else {
|
|
|
- file.localID = null;
|
|
|
+ sharedFileUpdated++;
|
|
|
+ // if user has downloaded the file on the device, avoid removing the
|
|
|
+ // localID reference.
|
|
|
+ // [Todo-fix: Excluded shared file's localIDs during syncALL]
|
|
|
+ remoteDiff.localID = existingFile.localID;
|
|
|
}
|
|
|
- final bool wasUploadedOnAPreviousInstallation =
|
|
|
- existingFiles.length == 1 && existingFiles[0].collectionID == null;
|
|
|
- if (wasUploadedOnAPreviousInstallation) {
|
|
|
- file.generatedID = existingFiles[0].generatedID;
|
|
|
- if (file.modificationTime != existingFiles[0].modificationTime) {
|
|
|
- // File was updated since the app was uninstalled
|
|
|
- // mark it for re-upload
|
|
|
- _logger.info(
|
|
|
- "re-upload because file was updated since last installation: "
|
|
|
- "remoteFile: ${file.toString()}, localFile: ${existingFiles[0].toString()}",
|
|
|
- );
|
|
|
- file.modificationTime = existingFiles[0].modificationTime;
|
|
|
- file.updationTime = null;
|
|
|
- updated++;
|
|
|
- } else {
|
|
|
- existing++;
|
|
|
- }
|
|
|
- toBeInserted.add(file);
|
|
|
+ toBeInserted.add(remoteDiff);
|
|
|
+ // end processing for file here, move to next file now
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ // If remoteDiff is not already synced (i.e. existingFile is null), check
|
|
|
+ // if the remoteFile was uploaded from this device.
|
|
|
+ // Note: DeviceFolder is ignored for iOS during matching
|
|
|
+ if (existingFile == null && remoteDiff.localID != null) {
|
|
|
+ final localFileEntries = await _db.getUnlinkedLocalMatchesForRemoteFile(
|
|
|
+ userID,
|
|
|
+ remoteDiff.localID,
|
|
|
+ remoteDiff.fileType,
|
|
|
+ title: remoteDiff.title,
|
|
|
+ deviceFolder: remoteDiff.deviceFolder,
|
|
|
+ );
|
|
|
+ if (localFileEntries.isEmpty) {
|
|
|
+ // set remote file's localID as null because corresponding local file
|
|
|
+ // does not exist [Case 2, do not retain localID of the remote file]
|
|
|
+ remoteDiff.localID = null;
|
|
|
} else {
|
|
|
- bool foundMatchingCollection = false;
|
|
|
- for (final existingFile in existingFiles) {
|
|
|
- if (file.collectionID == existingFile.collectionID &&
|
|
|
- file.uploadedFileID == existingFile.uploadedFileID) {
|
|
|
- // File was updated on remote
|
|
|
- if (file.creationTime != existingFile.creationTime) {
|
|
|
- hasAnyCreationTimeChanged = true;
|
|
|
- }
|
|
|
- foundMatchingCollection = true;
|
|
|
- file.generatedID = existingFile.generatedID;
|
|
|
- toBeInserted.add(file);
|
|
|
- await clearCache(file);
|
|
|
- localButUpdatedOnRemote++;
|
|
|
- break;
|
|
|
- }
|
|
|
+ // case 4: Check and schedule the file for update
|
|
|
+ final int maxModificationTime = localFileEntries
|
|
|
+ .map(
|
|
|
+ (e) => e.modificationTime ?? 0,
|
|
|
+ )
|
|
|
+ .reduce(max);
|
|
|
+ if (maxModificationTime > remoteDiff.modificationTime) {
|
|
|
+ localButUpdatedOnDevice++;
|
|
|
+ await FileUpdationDB.instance.insertMultiple(
|
|
|
+ [remoteDiff.localID],
|
|
|
+ FileUpdationDB.modificationTimeUpdated,
|
|
|
+ );
|
|
|
}
|
|
|
- if (!foundMatchingCollection) {
|
|
|
- // Added to a new collection
|
|
|
- toBeInserted.add(file);
|
|
|
- localButAddedToNewCollectionOnRemote++;
|
|
|
+
|
|
|
+ localFileEntries.removeWhere(
|
|
|
+ (e) =>
|
|
|
+ e.uploadedFileID != null ||
|
|
|
+ alreadyClaimedLocalFilesGenID.contains(e.generatedID),
|
|
|
+ );
|
|
|
+
|
|
|
+ if (localFileEntries.isNotEmpty) {
|
|
|
+ // file uploaded from same device, replace the local file row by
|
|
|
+ // setting the generated ID of remoteFile to localFile generatedID
|
|
|
+ existingFile = localFileEntries.first;
|
|
|
+ localUploadedFromDevice++;
|
|
|
+ alreadyClaimedLocalFilesGenID.add(existingFile.generatedID);
|
|
|
+ remoteDiff.generatedID = existingFile.generatedID;
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
+ if (existingFile != null &&
|
|
|
+ _shouldReloadHomeGallery(remoteDiff, existingFile)) {
|
|
|
+ needsGalleryReload = true;
|
|
|
+ } else {
|
|
|
+ remoteNewFile++;
|
|
|
+ }
|
|
|
+ toBeInserted.add(remoteDiff);
|
|
|
}
|
|
|
await _db.insertMultiple(toBeInserted);
|
|
|
_logger.info(
|
|
|
"Diff to be deduplicated was: " +
|
|
|
diff.length.toString() +
|
|
|
" out of which \n" +
|
|
|
- existing.toString() +
|
|
|
+ localUploadedFromDevice.toString() +
|
|
|
" was uploaded from device, \n" +
|
|
|
- updated.toString() +
|
|
|
+ localButUpdatedOnDevice.toString() +
|
|
|
" was uploaded from device, but has been updated since and should be reuploaded, \n" +
|
|
|
- remote.toString() +
|
|
|
- " was uploaded from remote, \n" +
|
|
|
- localButUpdatedOnRemote.toString() +
|
|
|
- " was uploaded from device but updated on remote, and \n" +
|
|
|
- localButAddedToNewCollectionOnRemote.toString() +
|
|
|
- " was uploaded from device but added to a new collection on remote.",
|
|
|
+ sharedFileNew.toString() +
|
|
|
+ " new sharedFiles, \n" +
|
|
|
+ sharedFileUpdated.toString() +
|
|
|
+ " updatedSharedFiles, and \n" +
|
|
|
+ remoteNewFile.toString() +
|
|
|
+ " remoteFiles seen first time",
|
|
|
);
|
|
|
- if (hasAnyCreationTimeChanged) {
|
|
|
+ if (needsGalleryReload) {
|
|
|
Bus.instance.fire(ForceReloadHomeGalleryEvent());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ bool _shouldClearCache(File remoteFile, File existingFile) {
|
|
|
+ if (remoteFile.hash != null && existingFile.hash != null) {
|
|
|
+ return remoteFile.hash != existingFile.hash;
|
|
|
+ }
|
|
|
+ return remoteFile.updationTime != (existingFile.updationTime ?? 0);
|
|
|
+ }
|
|
|
+
|
|
|
+ bool _shouldReloadHomeGallery(File remoteFile, File existingFile) {
|
|
|
+ int remoteCreationTime = remoteFile.creationTime;
|
|
|
+ if (remoteFile.pubMmdVersion > 0 &&
|
|
|
+ (remoteFile.pubMagicMetadata.editedTime ?? 0) != 0) {
|
|
|
+ remoteCreationTime = remoteFile.pubMagicMetadata.editedTime;
|
|
|
+ }
|
|
|
+ if (remoteCreationTime != existingFile.creationTime) {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ if (existingFile.mMdVersion > 0 &&
|
|
|
+ remoteFile.mMdVersion != existingFile.mMdVersion &&
|
|
|
+ remoteFile.magicMetadata.visibility !=
|
|
|
+ existingFile.magicMetadata.visibility) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
// return true if the client needs to re-sync the collections from previous
|
|
|
// version
|
|
|
- bool _hasReSynced() {
|
|
|
- return _prefs.containsKey(kHasSyncedEditTime) &&
|
|
|
- _prefs.containsKey(kHasSyncedArchiveKey);
|
|
|
+ bool _shouldResetSyncTime() {
|
|
|
+ return !_prefs.containsKey(kHasSyncedEditTime) ||
|
|
|
+ !_prefs.containsKey(kHasSyncedArchiveKey);
|
|
|
}
|
|
|
|
|
|
- Future<void> _markReSyncAsDone() async {
|
|
|
+ Future<void> _markResetSyncTimeAsDone() async {
|
|
|
await _prefs.setBool(kHasSyncedArchiveKey, true);
|
|
|
await _prefs.setBool(kHasSyncedEditTime, true);
|
|
|
+ // Check to avoid regression because of change or additions of keys
|
|
|
+ if (_shouldResetSyncTime()) {
|
|
|
+ throw Exception("_shouldResetSyncTime should return false now");
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
int _getSinceTimeForReSync() {
|