Sync trashed items in local db

lib/db/trash_db.dart

@@ -3,7 +3,9 @@ import 'dart:io';
 import 'package:logging/logging.dart';
 import 'package:path/path.dart';
 import 'package:path_provider/path_provider.dart';
+import 'package:photos/models/file_type.dart';
 import 'package:photos/models/magic_metadata.dart';
+import 'package:photos/models/trash_file.dart';
 import 'package:sqflite/sqflite.dart';
 
 class TrashDB {
@@ -30,7 +32,6 @@ class TrashDB {
   static final columnFileType = 'file_type';
   static final columnFileSubType = 'file_sub_type';
   static final columnDuration = 'duration';
-  static final columnExif = 'exif';
   static final columnHash = 'hash';
   static final columnMetadataVersion = 'metadata_version';
   static final columnModificationTime = 'modification_time';
@@ -97,9 +98,89 @@ class TrashDB {
       onCreate: _onCreate,
     );
   }
+
   Future<void> clearTable() async {
     final db = await instance.database;
     await db.delete(tableName);
   }
 
+  Future<void> insertMultiple(List<Trash> trashFiles) async {
+    final startTime = DateTime.now();
+    final db = await instance.database;
+    var batch = db.batch();
+    int batchCounter = 0;
+    for (Trash trash in trashFiles) {
+      if (batchCounter == 400) {
+        await batch.commit(noResult: true);
+        batch = db.batch();
+        batchCounter = 0;
+      }
+      batch.insert(
+        tableName,
+        _getRowForTrash(trash),
+        conflictAlgorithm: ConflictAlgorithm.replace,
+      );
+      batchCounter++;
+    }
+    await batch.commit(noResult: true);
+    final endTime = DateTime.now();
+    final duration = Duration(
+        microseconds:
+            endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch);
+    _logger.info("Batch insert of " +
+        trashFiles.length.toString() +
+        " took " +
+        duration.inMilliseconds.toString() +
+        "ms.");
+  }
+
+  Future<int> insert(Trash trash) async {
+    final db = await instance.database;
+    return db.insert(
+      tableName,
+      _getRowForTrash(trash),
+      conflictAlgorithm: ConflictAlgorithm.replace,
+    );
+  }
+
+  Future<int> delete(List<int> uploadedFileIDs) async {
+    final db = await instance.database;
+    return db.delete(
+      tableName,
+      where: '$columnUploadedFileID IN (${uploadedFileIDs.join(', ')})',
+    );
+  }
+
+  Map<String, dynamic> _getRowForTrash(Trash trash) {
+    final file = trash.file;
+    final row = <String, dynamic>{};
+    row[columnTrashUpdatedAt] = trash.updateAt;
+    row[columnTrashDeleteBy] = trash.deleteBy;
+    row[columnUploadedFileID] = file.uploadedFileID;
+    row[columnCollectionID] = file.collectionID;
+    row[columnOwnerID] = file.ownerID;
+    row[columnLocalID] = file.localID;
+    row[columnTitle] = file.title;
+    row[columnDeviceFolder] = file.deviceFolder;
+    if (file.location != null) {
+      row[columnLatitude] = file.location.latitude;
+      row[columnLongitude] = file.location.longitude;
+    }
+    row[columnFileType] = getInt(file.fileType);
+    row[columnCreationTime] = file.creationTime;
+    row[columnModificationTime] = file.modificationTime;
+    row[columnEncryptedKey] = file.encryptedKey;
+    row[columnKeyDecryptionNonce] = file.keyDecryptionNonce;
+    row[columnFileDecryptionHeader] = file.fileDecryptionHeader;
+    row[columnThumbnailDecryptionHeader] = file.thumbnailDecryptionHeader;
+    row[columnFileSubType] = file.fileSubType ?? -1;
+    row[columnDuration] = file.duration ?? 0;
+    row[columnHash] = file.hash;
+    row[columnMetadataVersion] = file.metadataVersion;
+    row[columnMMdVersion] = file.mMdVersion ?? 0;
+    row[columnMMdEncodedJson] = file.mMdEncodedJson ?? '{}';
+    row[columnMMdVisibility] =
+        file.magicMetadata?.visibility ?? kVisibilityVisible;
+    return row;
+  }
 }

lib/main.dart

@@ -4,6 +4,7 @@ import 'package:background_fetch/background_fetch.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_easyloading/flutter_easyloading.dart';
+import 'package:flutter_gen/gen_l10n/app_localizations.dart';
 import 'package:flutter_localizations/flutter_localizations.dart';
 import 'package:in_app_purchase/in_app_purchase.dart';
 import 'package:logging/logging.dart';
@@ -20,6 +21,7 @@ import 'package:photos/services/memories_service.dart';
 import 'package:photos/services/notification_service.dart';
 import 'package:photos/services/remote_sync_service.dart';
 import 'package:photos/services/sync_service.dart';
+import 'package:photos/services/trash_sync_service.dart';
 import 'package:photos/services/update_service.dart';
 import 'package:photos/ui/app_lock.dart';
 import 'package:photos/ui/home_widget.dart';
@@ -29,7 +31,6 @@ import 'package:photos/utils/file_uploader.dart';
 import 'package:photos/utils/local_settings.dart';
 import 'package:shared_preferences/shared_preferences.dart';
 import 'package:super_logging/super_logging.dart';
-import 'package:flutter_gen/gen_l10n/app_localizations.dart';
 
 import 'l10n/l10n.dart';
 
@@ -145,6 +146,7 @@ Future<void> _init(bool isBackground) async {
   await CollectionsService.instance.init();
   await FileUploader.instance.init(isBackground);
   await LocalSyncService.instance.init(isBackground);
+  await TrashSyncService.instance.init();
   await RemoteSyncService.instance.init();
   await SyncService.instance.init();
   await MemoriesService.instance.init();

lib/services/remote_sync_service.dart

@@ -15,6 +15,7 @@ import 'package:photos/models/file.dart';
 import 'package:photos/models/file_type.dart';
 import 'package:photos/services/collections_service.dart';
 import 'package:photos/services/local_sync_service.dart';
+import 'package:photos/services/trash_sync_service.dart';
 import 'package:photos/utils/diff_fetcher.dart';
 import 'package:photos/utils/file_uploader.dart';
 import 'package:photos/utils/file_util.dart';
@@ -61,6 +62,7 @@ class RemoteSyncService {
       await _markArchiveAsSynced();
     }
 
+    await TrashSyncService.instance.syncTrash();
     bool hasUploadedFiles = await _uploadDiff();
     if (hasUploadedFiles) {
       sync(silently: true);

lib/services/trash_sync_service.dart

@@ -1,15 +1,69 @@
 import 'package:dio/dio.dart';
+import 'package:logging/logging.dart';
 import 'package:photos/core/configuration.dart';
 import 'package:photos/core/network.dart';
+import 'package:photos/db/files_db.dart';
+import 'package:photos/db/trash_db.dart';
 import 'package:photos/models/trash_item_request.dart';
+import 'package:photos/utils/trash_diff_fetcher.dart';
+import 'package:shared_preferences/shared_preferences.dart';
 
 class TrashSyncService {
+  final _logger = Logger("TrashSyncService");
+  final _diffFetcher = TrashDiffFetcher();
+  final _filesDB = FilesDB.instance;
+  final _trashDB = TrashDB.instance;
+  static const kDiffLimit = 2500;
+  static const kLastTrashSyncTime = "last_trash_sync_time";
+  SharedPreferences _prefs;
+
   TrashSyncService._privateConstructor();
 
   static final TrashSyncService instance =
       TrashSyncService._privateConstructor();
   final _dio = Network.instance.getDio();
 
+  Future<void> init() async {
+    _prefs = await SharedPreferences.getInstance();
+  }
+
+  Future<void> syncTrash() async {
+    final lastSyncTime = getSyncTime();
+    _logger.fine('sync trash sinceTime : $lastSyncTime');
+    var diff = await _diffFetcher.getTrashFilesDiff(lastSyncTime, kDiffLimit);
+    if (diff.trashedFiles.isNotEmpty) {
+      _logger.fine("inserting ${diff.trashedFiles.length} items in trash");
+      await _trashDB.insertMultiple(diff.trashedFiles);
+    }
+    if (diff.deletedFiles.isNotEmpty) {
+      _logger.fine("discard ${diff.deletedFiles.length} deleted items");
+      await _trashDB
+          .delete(diff.deletedFiles.map((e) => e.file.uploadedFileID).toList());
+    }
+    if (diff.restoredFiles.isNotEmpty) {
+      _logger.fine("discard ${diff.restoredFiles.length} restored items");
+      await _trashDB.delete(
+          diff.restoredFiles.map((e) => e.file.uploadedFileID).toList());
+    }
+    if (diff.lastSyncedTimeStamp != 0) {
+      await setSyncTime(diff.lastSyncedTimeStamp);
+    }
+    if (diff.fetchCount == kDiffLimit) {
+      return await syncTrash();
+    }
+  }
+
+  Future<void> setSyncTime(int time) async {
+    if (time == null) {
+      return _prefs.remove(kLastTrashSyncTime);
+    }
+    return _prefs.setInt(kLastTrashSyncTime, time);
+  }
+
+  int getSyncTime() {
+    return _prefs.getInt(kLastTrashSyncTime) ?? 0;
+  }
+
   Future<void> trashFilesOnServer(List<TrashRequest> trashRequestItems) async {
     final params = <String, dynamic>{};
     params["items"] = [];

lib/utils/trash_diff_fetcher.dart

@@ -1,4 +1,5 @@
 import 'dart:convert';
+import 'dart:math';
 
 import 'package:dio/dio.dart';
 import 'package:flutter_sodium/flutter_sodium.dart';
@@ -6,10 +7,8 @@ import 'package:logging/logging.dart';
 import 'package:photos/core/configuration.dart';
 import 'package:photos/core/event_bus.dart';
 import 'package:photos/core/network.dart';
-import 'package:photos/db/files_db.dart';
 import 'package:photos/events/remote_sync_event.dart';
 import 'package:photos/models/file.dart';
-import 'package:photos/models/magic_metadata.dart';
 import 'package:photos/models/trash_file.dart';
 import 'package:photos/utils/crypto_util.dart';
 import 'package:photos/utils/file_download_util.dart';
@@ -29,6 +28,7 @@ class TrashDiffFetcher {
           "limit": limit,
         },
       );
+      int latestUpdatedAtTime = 0;
       final trashedFiles = <Trash>[];
       final deletedFiles = <Trash>[];
       final restoredFiles = <Trash>[];
@@ -40,6 +40,7 @@ class TrashDiffFetcher {
           final trash = Trash();
           trash.createdAt = item['createdAt'];
           trash.updateAt = item['updatedAt'];
+          latestUpdatedAtTime = max(latestUpdatedAtTime, trash.updateAt);
           trash.deleteBy = item['deleteBy'];
           trash.file = File();
           trash.file.uploadedFileID = item["file"]["id"];
@@ -70,8 +71,6 @@ class TrashDiffFetcher {
                 Sodium.base642bin(item["file"]['magicMetadata']['header']));
             trash.file.mMdEncodedJson = utf8.decode(utfEncodedMmd);
             trash.file.mMdVersion = item["file"]['magicMetadata']['version'];
-            trash.file.magicMetadata =
-                MagicMetadata.fromEncodedJson(trash.file.mMdEncodedJson);
           }
           if (item["isDeleted"]) {
             deletedFiles.add(trash);
@@ -93,10 +92,11 @@ class TrashDiffFetcher {
                         startTime.microsecondsSinceEpoch))
                 .inMilliseconds
                 .toString());
-        return Diff(trashedFiles, restoredFiles, deletedFiles, diff.length);
+        return Diff(trashedFiles, restoredFiles, deletedFiles, diff.length,
+            latestUpdatedAtTime);
       } else {
         Bus.instance.fire(RemoteSyncEvent(false));
-        return Diff(<Trash>[], <Trash>[], <Trash>[], 0);
+        return Diff(<Trash>[], <Trash>[], <Trash>[], 0, 0);
       }
     } catch (e, s) {
       _logger.severe(e, s);
@@ -110,7 +110,8 @@ class Diff {
   final List<Trash> restoredFiles;
   final List<Trash> deletedFiles;
   final int fetchCount;
+  final int lastSyncedTimeStamp;
 
   Diff(this.trashedFiles, this.restoredFiles, this.deletedFiles,
-      this.fetchCount);
+      this.fetchCount, this.lastSyncedTimeStamp);
 }