[mob][photos] Remove jank

mobile/lib/db/files_db.dart

@@ -695,6 +695,17 @@ class FilesDB {
     return files;
   }
 
+  Future<List<EnteFile>> getAllFilesFromCollections(
+    Iterable<int> collectionID,
+  ) async {
+    final db = await instance.sqliteAsyncDB;
+    final String sql =
+        'SELECT * FROM $filesTable WHERE $columnCollectionID IN (${collectionID.join(',')})';
+    final results = await db.getAll(sql);
+    final files = convertToFiles(results);
+    return files;
+  }
+
   Future<List<EnteFile>> getNewFilesInCollection(
     int collectionID,
     int addedTime,

mobile/lib/services/machine_learning/face_ml/face_ml_service.dart

@@ -574,11 +574,14 @@ class FaceMlService {
       isImageIndexRunning = true;
       _logger.info('starting image indexing');
 
+      final w = (kDebugMode ? EnteWatch('prepare indexing files') : null)
+        ?..start();
       final Map<int, int> alreadyIndexedFiles =
           await FaceMLDataDB.instance.getIndexedFileIds();
-      // w?.log('getIndexedFileIds');
+      w?.log('getIndexedFileIds');
       final List<EnteFile> enteFiles =
           await SearchService.instance.getAllFiles();
+      w?.log('getAllFiles');
 
       // Make sure the image conversion isolate is spawned
       // await ImageMlIsolate.instance.ensureSpawned();
@@ -589,11 +592,10 @@ class FaceMlService {
       final stopwatch = Stopwatch()..start();
       final List<EnteFile> filesWithLocalID = <EnteFile>[];
       final List<EnteFile> filesWithoutLocalID = <EnteFile>[];
-      final List<EnteFile> hiddenFiles = <EnteFile>[];
-      final List<int> allFileIDsToBeIndexed = await getIndexableFileIDs();
+      final List<EnteFile> hiddenFilesToIndex = <EnteFile>[];
+      w?.log('getIndexableFileIDs');
 
       for (final EnteFile enteFile in enteFiles) {
-        allFileIDsToBeIndexed.remove(enteFile.uploadedFileID);
         if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) {
           fileSkippedCount++;
           continue;
@@ -604,26 +606,24 @@ class FaceMlService {
           filesWithLocalID.add(enteFile);
         }
       }
-      if (allFileIDsToBeIndexed.isNotEmpty) {
-        _logger.info(
-          'Found ${allFileIDsToBeIndexed.length} files outside searchable files',
-        );
-        final List<EnteFile> uploadFiles =
-            await FilesDB.instance.getUploadedFiles(allFileIDsToBeIndexed);
-        for (final EnteFile enteFile in uploadFiles) {
-          if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) {
-            fileSkippedCount++;
-            continue;
-          }
-          hiddenFiles.add(enteFile);
+      w?.log('sifting through all normal files');
+      final List<EnteFile> hiddenFiles =
+          await SearchService.instance.getHiddenFiles();
+      w?.log('getHiddenFiles: ${hiddenFiles.length} hidden files');
+      for (final EnteFile enteFile in hiddenFiles) {
+        if (_skipAnalysisEnteFile(enteFile, alreadyIndexedFiles)) {
+          fileSkippedCount++;
+          continue;
         }
+        hiddenFilesToIndex.add(enteFile);
       }
 
       // list of files where files with localID are first
       final sortedBylocalID = <EnteFile>[];
       sortedBylocalID.addAll(filesWithLocalID);
       sortedBylocalID.addAll(filesWithoutLocalID);
-      sortedBylocalID.addAll(hiddenFiles);
+      sortedBylocalID.addAll(hiddenFilesToIndex);
+      w?.log('preparing all files to index');
       final List<List<EnteFile>> chunks =
           sortedBylocalID.chunks(_embeddingFetchLimit);
       outerLoop:

mobile/lib/services/search_service.dart

@@ -42,6 +42,7 @@ import 'package:tuple/tuple.dart';
 
 class SearchService {
   Future<List<EnteFile>>? _cachedFilesFuture;
+  Future<List<EnteFile>>? _cachedHiddenFilesFuture;
   final _logger = Logger((SearchService).toString());
   final _collectionService = CollectionsService.instance;
   static const _maximumResultsLimit = 20;
@@ -54,6 +55,7 @@ class SearchService {
     Bus.instance.on<LocalPhotosUpdatedEvent>().listen((event) {
       // only invalidate, let the load happen on demand
       _cachedFilesFuture = null;
+      _cachedHiddenFilesFuture = null;
     });
   }
 
@@ -73,8 +75,21 @@ class SearchService {
     return _cachedFilesFuture!;
   }
 
+  Future<List<EnteFile>> getHiddenFiles() async {
+    if (_cachedHiddenFilesFuture != null) {
+      return _cachedHiddenFilesFuture!;
+    }
+    _logger.fine("Reading hidden files from db");
+    final hiddenCollections =
+        CollectionsService.instance.getHiddenCollectionIds();
+    _cachedHiddenFilesFuture =
+        FilesDB.instance.getAllFilesFromCollections(hiddenCollections);
+    return _cachedHiddenFilesFuture!;
+  }
+
   void clearCache() {
     _cachedFilesFuture = null;
+    _cachedHiddenFilesFuture = null;
   }
 
   // getFilteredCollectionsWithThumbnail removes deleted or archived or