Merge pull request #125 from ente-io/collections_diff_v2

Collection Diff: switch to v2/diff

lib/services/remote_sync_service.dart

@@ -31,11 +31,12 @@ class RemoteSyncService {
   int _completedUploads = 0;
   SharedPreferences _prefs;
 
-  static const kDiffLimit = 2500;
   static const kHasSyncedArchiveKey = "has_synced_archive";
+
   // 28 Sept, 2021 9:03:20 AM IST
   static const kArchiveFeatureReleaseTime = 1632800000000000;
   static const kHasSyncedEditTime = "has_synced_edit_time";
+
   // 29 October, 2021 3:56:40 AM IST
   static const kEditTimeFeatureReleaseTime = 1635460000000000;
 
@@ -104,8 +105,8 @@ class RemoteSyncService {
   }
 
   Future<void> _syncCollectionDiff(int collectionID, int sinceTime) async {
-    final diff = await _diffFetcher.getEncryptedFilesDiff(
-        collectionID, sinceTime, kDiffLimit);
+    final diff =
+        await _diffFetcher.getEncryptedFilesDiff(collectionID, sinceTime);
     if (diff.deletedFiles.isNotEmpty) {
       final fileIDs = diff.deletedFiles.map((f) => f.uploadedFileID).toList();
       final deletedFiles =
@@ -126,7 +127,12 @@ class RemoteSyncService {
       Bus.instance
           .fire(CollectionUpdatedEvent(collectionID, diff.updatedFiles));
     }
-    if (diff.fetchCount == kDiffLimit) {
+
+    if (diff.latestUpdatedAtTime > 0) {
+      await _collectionsService.setCollectionSyncTime(
+          collectionID, diff.latestUpdatedAtTime);
+    }
+    if (diff.hasMore) {
       return await _syncCollectionDiff(collectionID,
           _collectionsService.getCollectionSyncTime(collectionID));
     }
@@ -330,10 +336,6 @@ class RemoteSyncService {
       }
     }
     await _db.insertMultiple(toBeInserted);
-    if (toBeInserted.isNotEmpty) {
-      await _collectionsService.setCollectionSyncTime(
-          collectionID, toBeInserted[toBeInserted.length - 1].updationTime);
-    }
     _logger.info(
       "Diff to be deduplicated was: " +
           diff.length.toString() +

lib/utils/diff_fetcher.dart

@@ -1,4 +1,5 @@
 import 'dart:convert';
+import 'dart:math';
 
 import 'package:dio/dio.dart';
 import 'package:flutter_sodium/flutter_sodium.dart';
@@ -17,23 +18,23 @@ class DiffFetcher {
   final _logger = Logger("DiffFetcher");
   final _dio = Network.instance.getDio();
 
-  Future<Diff> getEncryptedFilesDiff(
-      int collectionID, int sinceTime, int limit) async {
+  Future<Diff> getEncryptedFilesDiff(int collectionID, int sinceTime) async {
     try {
       final response = await _dio.get(
-        Configuration.instance.getHttpEndpoint() + "/collections/diff",
+        Configuration.instance.getHttpEndpoint() + "/collections/v2/diff",
         options: Options(
             headers: {"X-Auth-Token": Configuration.instance.getToken()}),
         queryParameters: {
           "collectionID": collectionID,
           "sinceTime": sinceTime,
-          "limit": limit,
         },
       );
       final files = <File>[];
+      int latestUpdatedAtTime = 0;
       if (response != null) {
         Bus.instance.fire(RemoteSyncEvent(true));
         final diff = response.data["diff"] as List;
+        final bool hasMore = response.data["hasMore"] as bool;
         final startTime = DateTime.now();
         final existingFiles =
             await FilesDB.instance.getUploadedFileIDs(collectionID);
@@ -42,6 +43,8 @@ class DiffFetcher {
           final file = File();
           file.uploadedFileID = item["id"];
           file.collectionID = item["collectionID"];
+          file.updationTime = item["updationTime"];
+          latestUpdatedAtTime = max(latestUpdatedAtTime, file.updationTime);
           if (item["isDeleted"]) {
             if (existingFiles.contains(file.uploadedFileID)) {
               deletedFiles.add(file);
@@ -55,7 +58,6 @@ class DiffFetcher {
               file.generatedID = existingFile.generatedID;
             }
           }
-          file.updationTime = item["updationTime"];
           file.ownerID = item["ownerID"];
           file.encryptedKey = item["encryptedKey"];
           file.keyDecryptionNonce = item["keyDecryptionNonce"];
@@ -105,10 +107,10 @@ class DiffFetcher {
                         startTime.microsecondsSinceEpoch))
                 .inMilliseconds
                 .toString());
-        return Diff(files, deletedFiles, diff.length);
+        return Diff(files, deletedFiles, hasMore, latestUpdatedAtTime);
       } else {
         Bus.instance.fire(RemoteSyncEvent(false));
-        return Diff(<File>[], <File>[], 0);
+        return Diff(<File>[], <File>[], false, 0);
       }
     } catch (e, s) {
       _logger.severe(e, s);
@@ -120,7 +122,9 @@ class DiffFetcher {
 class Diff {
   final List<File> updatedFiles;
   final List<File> deletedFiles;
-  final int fetchCount;
+  final bool hasMore;
+  final int latestUpdatedAtTime;
 
-  Diff(this.updatedFiles, this.deletedFiles, this.fetchCount);
+  Diff(this.updatedFiles, this.deletedFiles, this.hasMore,
+      this.latestUpdatedAtTime);
 }

pubspec.yaml

@@ -11,7 +11,7 @@ description: ente photos application
 # In iOS, build-name is used as CFBundleShortVersionString while build-number used as CFBundleVersion.
 # Read more about iOS versioning at
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
-version: 0.3.40+250
+version: 0.3.41+251
 
 environment:
   sdk: ">=2.10.0 <3.0.0"