Merge pull request #649 from ente-io/batch-files

Batch files

lib/core/constants.dart

@@ -14,6 +14,7 @@ const int android11SDKINT = 30;
 const int jan011981Time = 347155200000000;
 const int galleryLoadStartTime = -8000000000000000; // Wednesday, March 6, 1748
 const int galleryLoadEndTime = 9223372036854775807; // 2^63 -1
+const int batchSize = 1000;
 
 // used to identify which ente file are available in app cache
 // todo: 6Jun22: delete old media identifier after 3 months

lib/services/collections_service.dart

@@ -10,6 +10,7 @@ import 'package:flutter/foundation.dart';
 import 'package:flutter_sodium/flutter_sodium.dart';
 import 'package:logging/logging.dart';
 import 'package:photos/core/configuration.dart';
+import 'package:photos/core/constants.dart';
 import 'package:photos/core/errors.dart';
 import 'package:photos/core/event_bus.dart';
 import 'package:photos/core/network.dart';
@@ -21,6 +22,7 @@ import 'package:photos/events/collection_updated_event.dart';
 import 'package:photos/events/files_updated_event.dart';
 import 'package:photos/events/force_reload_home_gallery_event.dart';
 import 'package:photos/events/local_photos_updated_event.dart';
+import 'package:photos/extensions/list.dart';
 import 'package:photos/extensions/stop_watch.dart';
 import 'package:photos/models/api/collection/create_request.dart';
 import 'package:photos/models/collection.dart';
@@ -719,35 +721,37 @@ class CollectionsService {
 
     final params = <String, dynamic>{};
     params["collectionID"] = collectionID;
-    for (final file in files) {
-      final key = decryptFileKey(file);
-      file.generatedID = null; // So that a new entry is created in the FilesDB
-      file.collectionID = collectionID;
-      final encryptedKeyData =
-          CryptoUtil.encryptSync(key, getCollectionKey(collectionID));
-      file.encryptedKey = Sodium.bin2base64(encryptedKeyData.encryptedData);
-      file.keyDecryptionNonce = Sodium.bin2base64(encryptedKeyData.nonce);
-      if (params["files"] == null) {
-        params["files"] = [];
+    final batchedFiles = files.chunks(batchSize);
+    for (final batch in batchedFiles) {
+      params["files"] = [];
+      for (final file in batch) {
+        final key = decryptFileKey(file);
+        file.generatedID =
+            null; // So that a new entry is created in the FilesDB
+        file.collectionID = collectionID;
+        final encryptedKeyData =
+            CryptoUtil.encryptSync(key, getCollectionKey(collectionID));
+        file.encryptedKey = Sodium.bin2base64(encryptedKeyData.encryptedData);
+        file.keyDecryptionNonce = Sodium.bin2base64(encryptedKeyData.nonce);
+        params["files"].add(
+          CollectionFileItem(
+            file.uploadedFileID,
+            file.encryptedKey,
+            file.keyDecryptionNonce,
+          ).toMap(),
+        );
       }
-      params["files"].add(
-        CollectionFileItem(
-          file.uploadedFileID,
-          file.encryptedKey,
-          file.keyDecryptionNonce,
-        ).toMap(),
-      );
-    }
 
-    try {
-      await _enteDio.post(
-        "/collections/add-files",
-        data: params,
-      );
-      await _filesDB.insertMultiple(files);
-      Bus.instance.fire(CollectionUpdatedEvent(collectionID, files, "addTo"));
-    } catch (e) {
-      rethrow;
+      try {
+        await _enteDio.post(
+          "/collections/add-files",
+          data: params,
+        );
+        await _filesDB.insertMultiple(batch);
+        Bus.instance.fire(CollectionUpdatedEvent(collectionID, batch, "addTo"));
+      } catch (e) {
+        rethrow;
+      }
     }
   }
 
@@ -796,51 +800,55 @@ class CollectionsService {
   Future<void> restore(int toCollectionID, List<File> files) async {
     final params = <String, dynamic>{};
     params["collectionID"] = toCollectionID;
-    params["files"] = [];
     final toCollectionKey = getCollectionKey(toCollectionID);
-    for (final file in files) {
-      final key = decryptFileKey(file);
-      file.generatedID = null; // So that a new entry is created in the FilesDB
-      file.collectionID = toCollectionID;
-      final encryptedKeyData = CryptoUtil.encryptSync(key, toCollectionKey);
-      file.encryptedKey = Sodium.bin2base64(encryptedKeyData.encryptedData);
-      file.keyDecryptionNonce = Sodium.bin2base64(encryptedKeyData.nonce);
-      params["files"].add(
-        CollectionFileItem(
-          file.uploadedFileID,
-          file.encryptedKey,
-          file.keyDecryptionNonce,
-        ).toMap(),
-      );
-    }
-    try {
-      await _enteDio.post(
-        "/collections/restore-files",
-        data: params,
-      );
-      await _filesDB.insertMultiple(files);
-      await TrashDB.instance
-          .delete(files.map((e) => e.uploadedFileID).toList());
-      Bus.instance.fire(
-        CollectionUpdatedEvent(toCollectionID, files, "restore"),
-      );
-      Bus.instance.fire(FilesUpdatedEvent(files, source: "restore"));
-      // Remove imported local files which are imported but not uploaded.
-      // This handles the case where local file was trashed -> imported again
-      // but not uploaded automatically as it was trashed.
-      final localIDs = files
-          .where((e) => e.localID != null)
-          .map((e) => e.localID)
-          .toSet()
-          .toList();
-      if (localIDs.isNotEmpty) {
-        await _filesDB.deleteUnSyncedLocalFiles(localIDs);
+    final batchedFiles = files.chunks(batchSize);
+    for (final batch in batchedFiles) {
+      params["files"] = [];
+      for (final file in batch) {
+        final key = decryptFileKey(file);
+        file.generatedID =
+            null; // So that a new entry is created in the FilesDB
+        file.collectionID = toCollectionID;
+        final encryptedKeyData = CryptoUtil.encryptSync(key, toCollectionKey);
+        file.encryptedKey = Sodium.bin2base64(encryptedKeyData.encryptedData);
+        file.keyDecryptionNonce = Sodium.bin2base64(encryptedKeyData.nonce);
+        params["files"].add(
+          CollectionFileItem(
+            file.uploadedFileID,
+            file.encryptedKey,
+            file.keyDecryptionNonce,
+          ).toMap(),
+        );
+      }
+      try {
+        await _enteDio.post(
+          "/collections/restore-files",
+          data: params,
+        );
+        await _filesDB.insertMultiple(batch);
+        await TrashDB.instance
+            .delete(batch.map((e) => e.uploadedFileID).toList());
+        Bus.instance.fire(
+          CollectionUpdatedEvent(toCollectionID, batch, "restore"),
+        );
+        Bus.instance.fire(FilesUpdatedEvent(batch, source: "restore"));
+        // Remove imported local files which are imported but not uploaded.
+        // This handles the case where local file was trashed -> imported again
+        // but not uploaded automatically as it was trashed.
+        final localIDs = batch
+            .where((e) => e.localID != null)
+            .map((e) => e.localID)
+            .toSet()
+            .toList();
+        if (localIDs.isNotEmpty) {
+          await _filesDB.deleteUnSyncedLocalFiles(localIDs);
+        }
+        // Force reload home gallery to pull in the restored files
+        Bus.instance.fire(ForceReloadHomeGalleryEvent("restoredFromTrash"));
+      } catch (e, s) {
+        _logger.severe("failed to restore files", e, s);
+        rethrow;
       }
-      // Force reload home gallery to pull in the restored files
-      Bus.instance.fire(ForceReloadHomeGalleryEvent("restoredFromTrash"));
-    } catch (e, s) {
-      _logger.severe("failed to restore files", e, s);
-      rethrow;
     }
   }
 
@@ -858,27 +866,31 @@ class CollectionsService {
     final params = <String, dynamic>{};
     params["toCollectionID"] = toCollectionID;
     params["fromCollectionID"] = fromCollectionID;
-    params["files"] = [];
-    for (final file in files) {
-      final fileKey = decryptFileKey(file);
-      file.generatedID = null; // So that a new entry is created in the FilesDB
-      file.collectionID = toCollectionID;
-      final encryptedKeyData =
-          CryptoUtil.encryptSync(fileKey, getCollectionKey(toCollectionID));
-      file.encryptedKey = Sodium.bin2base64(encryptedKeyData.encryptedData);
-      file.keyDecryptionNonce = Sodium.bin2base64(encryptedKeyData.nonce);
-      params["files"].add(
-        CollectionFileItem(
-          file.uploadedFileID,
-          file.encryptedKey,
-          file.keyDecryptionNonce,
-        ).toMap(),
+    final batchedFiles = files.chunks(batchSize);
+    for (final batch in batchedFiles) {
+      params["files"] = [];
+      for (final file in batch) {
+        final fileKey = decryptFileKey(file);
+        file.generatedID =
+            null; // So that a new entry is created in the FilesDB
+        file.collectionID = toCollectionID;
+        final encryptedKeyData =
+            CryptoUtil.encryptSync(fileKey, getCollectionKey(toCollectionID));
+        file.encryptedKey = Sodium.bin2base64(encryptedKeyData.encryptedData);
+        file.keyDecryptionNonce = Sodium.bin2base64(encryptedKeyData.nonce);
+        params["files"].add(
+          CollectionFileItem(
+            file.uploadedFileID,
+            file.encryptedKey,
+            file.keyDecryptionNonce,
+          ).toMap(),
+        );
+      }
+      await _enteDio.post(
+        "/collections/move-files",
+        data: params,
       );
     }
-    await _enteDio.post(
-      "/collections/move-files",
-      data: params,
-    );
 
     // remove files from old collection
     await _filesDB.removeFromCollection(
@@ -929,20 +941,22 @@ class CollectionsService {
   Future<void> removeFromCollection(int collectionID, List<File> files) async {
     final params = <String, dynamic>{};
     params["collectionID"] = collectionID;
-    for (final file in files) {
-      if (params["fileIDs"] == null) {
-        params["fileIDs"] = <int>[];
+    final batchedFiles = files.chunks(batchSize);
+    for (final batch in batchedFiles) {
+      params["fileIDs"] = <int>[];
+      for (final file in batch) {
+        params["fileIDs"].add(file.uploadedFileID);
       }
-      params["fileIDs"].add(file.uploadedFileID);
+      await _enteDio.post(
+        "/collections/v2/remove-files",
+        data: params,
+      );
+
+      await _filesDB.removeFromCollection(collectionID, params["fileIDs"]);
+      Bus.instance
+          .fire(CollectionUpdatedEvent(collectionID, batch, "removeFrom"));
+      Bus.instance.fire(LocalPhotosUpdatedEvent(batch, source: "removeFrom"));
     }
-    await _enteDio.post(
-      "/collections/v2/remove-files",
-      data: params,
-    );
-    await _filesDB.removeFromCollection(collectionID, params["fileIDs"]);
-    Bus.instance
-        .fire(CollectionUpdatedEvent(collectionID, files, "removeFrom"));
-    Bus.instance.fire(LocalPhotosUpdatedEvent(files, source: "removeFrom"));
     RemoteSyncService.instance.sync(silently: true).ignore();
   }
 

lib/services/file_magic_service.dart

@@ -6,12 +6,14 @@ import 'package:dio/dio.dart';
 import 'package:flutter_sodium/flutter_sodium.dart';
 import 'package:logging/logging.dart';
 import 'package:photos/core/configuration.dart';
+import 'package:photos/core/constants.dart';
 import 'package:photos/core/event_bus.dart';
 import 'package:photos/core/network.dart';
 import 'package:photos/db/files_db.dart';
 import 'package:photos/events/files_updated_event.dart';
 import 'package:photos/events/force_reload_home_gallery_event.dart';
 import 'package:photos/events/local_photos_updated_event.dart';
+import 'package:photos/extensions/list.dart';
 import 'package:photos/models/file.dart';
 import 'package:photos/models/magic_metadata.dart';
 import 'package:photos/services/remote_sync_service.dart';
@@ -132,53 +134,57 @@ class FileMagicService {
     Map<String, dynamic> newMetadataUpdate,
   ) async {
     final params = <String, dynamic>{};
-    params['metadataList'] = [];
     final int ownerID = Configuration.instance.getUserID();
+    final batchedFiles = files.chunks(batchSize);
     try {
-      for (final file in files) {
-        if (file.uploadedFileID == null) {
-          throw AssertionError(
-            "operation is only supported on backed up files",
+      for (final batch in batchedFiles) {
+        params['metadataList'] = [];
+        for (final file in batch) {
+          if (file.uploadedFileID == null) {
+            throw AssertionError(
+              "operation is only supported on backed up files",
+            );
+          } else if (file.ownerID != ownerID) {
+            throw AssertionError("cannot modify memories not owned by you");
+          }
+          // read the existing magic metadata and apply new updates to existing data
+          // current update is simple replace. This will be enhanced in the future,
+          // as required.
+          final Map<String, dynamic> jsonToUpdate =
+              jsonDecode(file.mMdEncodedJson);
+          newMetadataUpdate.forEach((key, value) {
+            jsonToUpdate[key] = value;
+          });
+
+          // update the local information so that it's reflected on UI
+          file.mMdEncodedJson = jsonEncode(jsonToUpdate);
+          file.magicMetadata = MagicMetadata.fromJson(jsonToUpdate);
+
+          final fileKey = decryptFileKey(file);
+          final encryptedMMd = await CryptoUtil.encryptChaCha(
+            utf8.encode(jsonEncode(jsonToUpdate)),
+            fileKey,
           );
-        } else if (file.ownerID != ownerID) {
-          throw AssertionError("cannot modify memories not owned by you");
+          params['metadataList'].add(
+            UpdateMagicMetadataRequest(
+              id: file.uploadedFileID,
+              magicMetadata: MetadataRequest(
+                version: file.mMdVersion,
+                count: jsonToUpdate.length,
+                data: Sodium.bin2base64(encryptedMMd.encryptedData),
+                header: Sodium.bin2base64(encryptedMMd.header),
+              ),
+            ),
+          );
+          file.mMdVersion = file.mMdVersion + 1;
         }
-        // read the existing magic metadata and apply new updates to existing data
-        // current update is simple replace. This will be enhanced in the future,
-        // as required.
-        final Map<String, dynamic> jsonToUpdate =
-            jsonDecode(file.mMdEncodedJson);
-        newMetadataUpdate.forEach((key, value) {
-          jsonToUpdate[key] = value;
-        });
-
-        // update the local information so that it's reflected on UI
-        file.mMdEncodedJson = jsonEncode(jsonToUpdate);
-        file.magicMetadata = MagicMetadata.fromJson(jsonToUpdate);
 
-        final fileKey = decryptFileKey(file);
-        final encryptedMMd = await CryptoUtil.encryptChaCha(
-          utf8.encode(jsonEncode(jsonToUpdate)),
-          fileKey,
-        );
-        params['metadataList'].add(
-          UpdateMagicMetadataRequest(
-            id: file.uploadedFileID,
-            magicMetadata: MetadataRequest(
-              version: file.mMdVersion,
-              count: jsonToUpdate.length,
-              data: Sodium.bin2base64(encryptedMMd.encryptedData),
-              header: Sodium.bin2base64(encryptedMMd.header),
-            ),
-          ),
-        );
-        file.mMdVersion = file.mMdVersion + 1;
+        await _enteDio.put("/files/magic-metadata", data: params);
+        await _filesDB.insertMultiple(files);
       }
 
-      await _enteDio.put("/files/magic-metadata", data: params);
       // update the state of the selected file. Same file in other collection
       // should be eventually synced after remote sync has completed
-      await _filesDB.insertMultiple(files);
       RemoteSyncService.instance.sync(silently: true);
     } on DioError catch (e) {
       if (e.response != null && e.response.statusCode == 409) {

lib/services/trash_sync_service.dart

@@ -4,12 +4,14 @@ import 'dart:async';
 
 import 'package:dio/dio.dart';
 import 'package:logging/logging.dart';
+import 'package:photos/core/constants.dart';
 import 'package:photos/core/event_bus.dart';
 import 'package:photos/core/network.dart';
 import 'package:photos/db/trash_db.dart';
 import 'package:photos/events/collection_updated_event.dart';
 import 'package:photos/events/force_reload_trash_page_event.dart';
 import 'package:photos/events/trash_updated_event.dart';
+import 'package:photos/extensions/list.dart';
 import 'package:photos/models/file.dart';
 import 'package:photos/models/ignored_file.dart';
 import 'package:photos/models/trash_file.dart';
@@ -23,7 +25,6 @@ class TrashSyncService {
   final _diffFetcher = TrashDiffFetcher();
   final _trashDB = TrashDB.instance;
   static const kLastTrashSyncTime = "last_trash_sync_time";
-  static const kTrashBatchSize = 999;
   SharedPreferences _prefs;
 
   TrashSyncService._privateConstructor();
@@ -113,20 +114,14 @@ class TrashSyncService {
         includedFileIDs.add(item.fileID);
       }
     }
-    int currentBatchSize = 0;
     final requestData = <String, dynamic>{};
-    requestData["items"] = [];
-    for (final item in uniqueItems) {
-      currentBatchSize++;
-      requestData["items"].add(item.toJson());
-      if (currentBatchSize >= kTrashBatchSize) {
-        await _trashFiles(requestData);
-        requestData["items"] = [];
-        currentBatchSize = 0;
+    final batchedItems = uniqueItems.chunks(batchSize);
+    for (final batch in batchedItems) {
+      requestData["items"] = [];
+      for (final item in batch) {
+        requestData["items"].add(item.toJson());
       }
-    }
-    if (currentBatchSize > 0) {
-      return await _trashFiles(requestData);
+      await _trashFiles(requestData);
     }
   }
 
@@ -142,23 +137,26 @@ class TrashSyncService {
   Future<void> deleteFromTrash(List<File> files) async {
     final params = <String, dynamic>{};
     final uniqueFileIds = files.map((e) => e.uploadedFileID).toSet().toList();
-    params["fileIDs"] = [];
-    for (final fileID in uniqueFileIds) {
-      params["fileIDs"].add(fileID);
-    }
-    try {
-      await _enteDio.post(
-        "/trash/delete",
-        data: params,
-      );
-      await _trashDB.delete(uniqueFileIds);
-      Bus.instance.fire(TrashUpdatedEvent());
-      // no need to await on syncing trash from remote
-      unawaited(syncTrash());
-    } catch (e, s) {
-      _logger.severe("failed to delete from trash", e, s);
-      rethrow;
+    final batchedFileIDs = uniqueFileIds.chunks(batchSize);
+    for (final batch in batchedFileIDs) {
+      params["fileIDs"] = [];
+      for (final fileID in batch) {
+        params["fileIDs"].add(fileID);
+      }
+      try {
+        await _enteDio.post(
+          "/trash/delete",
+          data: params,
+        );
+        await _trashDB.delete(batch);
+        Bus.instance.fire(TrashUpdatedEvent());
+      } catch (e, s) {
+        _logger.severe("failed to delete from trash", e, s);
+        rethrow;
+      }
     }
+    // no need to await on syncing trash from remote
+    unawaited(syncTrash());
   }
 
   Future<void> emptyTrash() async {

lib/ui/viewer/gallery/gallery_overlay_widget.dart

@@ -271,11 +271,9 @@ class _OverlayWidgetState extends State<OverlayWidget> {
     if (Configuration.instance.hasConfiguredAccount() &&
         widget.type != GalleryType.sharedCollection &&
         widget.type != GalleryType.hidden) {
-      String msg = "Add to album";
       IconData iconData = Platform.isAndroid ? Icons.add : CupertinoIcons.add;
       // show upload icon instead of add for files selected in local gallery
       if (widget.type == GalleryType.localFolder) {
-        msg = "Upload to album";
         iconData = Icons.cloud_upload_outlined;
       }
       actions.add(