Merge pull request #117 from ente-io/ignored_v2

Ignored File: Use device folder & title for identifying ignored files on Android

lib/db/ignored_files_db.dart

@@ -1,6 +1,7 @@
 import 'dart:io';
-import 'package:path/path.dart';
+
 import 'package:logging/logging.dart';
+import 'package:path/path.dart';
 import 'package:path_provider/path_provider.dart';
 import 'package:photos/models/ignored_file.dart';
 import 'package:sqflite/sqflite.dart';
@@ -17,6 +18,7 @@ class IgnoredFilesDB {
 
   static final columnLocalID = 'local_id';
   static final columnTitle = 'title';
+  static final columnDeviceFolder = 'device_folder';
   static final columnReason = 'reason';
 
   Future _onCreate(Database db, int version) async {
@@ -24,10 +26,12 @@ class IgnoredFilesDB {
         CREATE TABLE $tableName (
           $columnLocalID TEXT NOT NULL,
           $columnTitle TEXT NOT NULL,
+          $columnDeviceFolder TEXT NOT NULL,
           $columnReason TEXT DEFAULT $kIgnoreReasonTrash,
-          UNIQUE($columnLocalID, $columnTitle)
+          UNIQUE($columnLocalID, $columnTitle, $columnDeviceFolder)
         );
       CREATE INDEX IF NOT EXISTS local_id_index ON $tableName($columnLocalID);
+      CREATE INDEX IF NOT EXISTS device_folder_index ON $tableName($columnDeviceFolder);
       ''');
   }
 
@@ -83,11 +87,8 @@ class IgnoredFilesDB {
     final duration = Duration(
         microseconds:
             endTime.microsecondsSinceEpoch - startTime.microsecondsSinceEpoch);
-    _logger.info("Batch insert of " +
-        ignoredFiles.length.toString() +
-        " took " +
-        duration.inMilliseconds.toString() +
-        "ms.");
+    _logger.info("Batch insert of ${ignoredFiles.length} "
+        "took ${duration.inMilliseconds} ms.");
   }
 
   Future<int> insert(IgnoredFile ignoredFile) async {
@@ -99,26 +100,45 @@ class IgnoredFilesDB {
     );
   }
 
-  // return map of localID to set of titles associated with the given localIDs
-  // Note: localIDs can easily clash across devices for Android, so we should
-  // always compare both localID & title in Android before ignoring the file for upload.
-  // iOS: localID is usually UUID and the title in localDB may be missing (before upload) as the
-  // photo manager library doesn't always fetch the title by default.
-  Future<Map<String, Set<String>>> getIgnoredFiles() async {
-    final db = await instance.database;
-    final rows = await db.query(tableName);
+  // returns a  map of device folder to set of title/filenames which exist
+  // in the particular device folder.
+  Future<Map<String, Set<String>>> getFilenamesForDeviceFolders(
+      Set<String> folders) async {
     final result = <String, Set<String>>{};
+    final db = await instance.database;
+
+    if (folders.isEmpty) {
+      return result;
+    }
+    String inParam = "";
+    for (final folder in folders) {
+      inParam += "'" + folder.replaceAll("'", "''") + "',";
+    }
+    inParam = inParam.substring(0, inParam.length - 1);
+    final rows =
+        await db.query(tableName, where: '$columnDeviceFolder IN ($inParam)');
     for (final row in rows) {
       final ignoredFile = _getIgnoredFileFromRow(row);
       result
-          .putIfAbsent(ignoredFile.localID, () => <String>{})
+          .putIfAbsent(ignoredFile.deviceFolder, () => <String>{})
           .add(ignoredFile.title);
     }
     return result;
   }
 
+  Future<Set<String>> getAllLocalIDs() async {
+    final db = await instance.database;
+    final rows = await db.query(tableName);
+    final result = <String>{};
+    for (final row in rows) {
+      result.add(row[columnLocalID]);
+    }
+    return result;
+  }
+
   IgnoredFile _getIgnoredFileFromRow(Map<String, dynamic> row) {
-    return IgnoredFile(row[columnLocalID], row[columnTitle], row[columnReason]);
+    return IgnoredFile(row[columnLocalID], row[columnTitle],
+        row[columnDeviceFolder], row[columnReason]);
   }
 
   Map<String, dynamic> _getRowForIgnoredFile(IgnoredFile ignoredFile) {
@@ -127,6 +147,7 @@ class IgnoredFilesDB {
     final row = <String, dynamic>{};
     row[columnLocalID] = ignoredFile.localID;
     row[columnTitle] = ignoredFile.title;
+    row[columnDeviceFolder] = ignoredFile.deviceFolder;
     row[columnReason] = ignoredFile.reason;
     return row;
   }

lib/models/ignored_file.dart

@@ -6,19 +6,23 @@ const kIgnoreReasonInvalidFile = "invalidFile";
 class IgnoredFile {
   final String localID;
   final String title;
+  final String deviceFolder;
   String reason;
 
-  IgnoredFile(this.localID, this.title, this.reason);
+  IgnoredFile(this.localID, this.title, this.deviceFolder, this.reason);
 
   factory IgnoredFile.fromTrashItem(TrashFile trashFile) {
     if (trashFile == null) return null;
     if (trashFile.localID == null ||
-        trashFile.title == null ||
         trashFile.localID.isEmpty ||
-        trashFile.title.isEmpty) {
+        trashFile.title == null ||
+        trashFile.title.isEmpty ||
+        trashFile.deviceFolder == null ||
+        trashFile.deviceFolder.isEmpty) {
       return null;
     }
 
-    return IgnoredFile(trashFile.localID, trashFile.title, kIgnoreReasonTrash);
+    return IgnoredFile(trashFile.localID, trashFile.title,
+        trashFile.deviceFolder, kIgnoreReasonTrash);
   }
 }

lib/services/remote_sync_service.dart

@@ -127,21 +127,24 @@ class RemoteSyncService {
     }
   }
 
+  // This method checks for deviceFolder + title for Android.
+  // For iOS, we rely on localIDs as they are uuid as title or deviceFolder (aka
+  // album name) can be missing due to various reasons.
   bool _shouldIgnoreFileUpload(
-      Map<String, Set<String>> ignoredFilesMap, File file) {
+    File file, {
+    Map<String, Set<String>> ignoredFilesMap,
+    Set<String> ignoredLocalIDs,
+  }) {
     if (file.localID == null || file.localID.isEmpty) {
       return false;
     }
-    if (!ignoredFilesMap.containsKey(file.localID)) {
-      return false;
-    }
-    // only compare title in Android because title may be missing in IOS
-    // and iOS anyways use uuid for localIDs of file, so collision should be
-    // rare.
-    if (Platform.isAndroid) {
-      return ignoredFilesMap[file.localID].contains(file.title ?? '');
+    if (Platform.isIOS) {
+      return ignoredLocalIDs.contains(file.localID);
     }
-    return true;
+    // For android, check if there's any ignored file with same device folder
+    // and title.
+    return ignoredFilesMap.containsKey(file.deviceFolder) &&
+        ignoredFilesMap[file.deviceFolder].contains(file.title);
   }
 
   Future<bool> _uploadDiff() async {
@@ -159,10 +162,17 @@ class RemoteSyncService {
           .removeWhere((element) => element.fileType == FileType.video);
     }
     if (filesToBeUploaded.isNotEmpty) {
-      final ignoredFilesMap = await IgnoredFilesDB.instance.getIgnoredFiles();
       final int prevCount = filesToBeUploaded.length;
-      filesToBeUploaded.removeWhere(
-          (file) => _shouldIgnoreFileUpload(ignoredFilesMap, file));
+      if (Platform.isAndroid) {
+        final ignoredFilesMap = await IgnoredFilesDB.instance
+            .getFilenamesForDeviceFolders(foldersToBackUp);
+        filesToBeUploaded.removeWhere((file) =>
+            _shouldIgnoreFileUpload(file, ignoredFilesMap: ignoredFilesMap));
+      } else {
+        final ignoredLocalIDs = await IgnoredFilesDB.instance.getAllLocalIDs();
+        filesToBeUploaded.removeWhere((file) =>
+            _shouldIgnoreFileUpload(file, ignoredLocalIDs: ignoredLocalIDs));
+      }
       if (prevCount != filesToBeUploaded.length) {
         _logger.info((prevCount - filesToBeUploaded.length).toString() +
             " files were ignored for upload");