[mob][photos] Cleanup face debug options

mobile/lib/face/db.dart

@@ -576,60 +576,6 @@ class FaceMLDataDB {
     }
   }
 
-  /// Returns a map of faceID to record of clusterId and faceEmbeddingBlob
-  ///
-  /// Only selects faces with score greater than [minScore] and blur score greater than [minClarity]
-  Future<Map<String, (int?, Uint8List)>> getFaceEmbeddingMap({
-    double minScore = kMinimumQualityFaceScore,
-    int minClarity = kLaplacianHardThreshold,
-    int maxFaces = 20000,
-    int offset = 0,
-    int batchSize = 10000,
-  }) async {
-    final EnteWatch w = EnteWatch("getFaceEmbeddingMap")..start();
-    w.logAndReset(
-      'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
-    );
-    final db = await instance.asyncDB;
-
-    final Map<String, (int?, Uint8List)> result = {};
-    while (true) {
-      // Query a batch of rows
-      final List<Map<String, dynamic>> maps = await db.getAll(
-        'SELECT $faceIDColumn, $faceEmbeddingBlob FROM $facesTable'
-        ' WHERE $faceScore > $minScore AND $faceBlur > $minClarity'
-        ' ORDER BY $faceIDColumn'
-        ' DESC LIMIT $batchSize OFFSET $offset',
-        // facesTable,
-        // columns: [faceIDColumn, faceEmbeddingBlob],
-        // where: '$faceScore > $minScore and $faceBlur > $minClarity',
-        // limit: batchSize,
-        // offset: offset,
-        // orderBy: '$faceIDColumn DESC',
-      );
-      // Break the loop if no more rows
-      if (maps.isEmpty) {
-        break;
-      }
-      final List<String> faceIds = [];
-      for (final map in maps) {
-        faceIds.add(map[faceIDColumn] as String);
-      }
-      final faceIdToClusterId = await getFaceIdsToClusterIds(faceIds);
-      for (final map in maps) {
-        final faceID = map[faceIDColumn] as String;
-        result[faceID] =
-            (faceIdToClusterId[faceID], map[faceEmbeddingBlob] as Uint8List);
-      }
-      if (result.length >= maxFaces) {
-        break;
-      }
-      offset += batchSize;
-    }
-    w.stopWithLog('done reading face embeddings ${result.length}');
-    return result;
-  }
-
   Future<Map<String, Uint8List>> getFaceEmbeddingMapForFile(
     List<int> fileIDs,
   ) async {
@@ -750,6 +696,7 @@ class FaceMLDataDB {
     return maps.first['count'] as int;
   }
 
+  /// WARNING: This method does not drop the persons and other feedback. Consider using [dropClustersAndPersonTable] instead.
   Future<void> resetClusterIDs() async {
     try {
       final db = await instance.asyncDB;
@@ -972,12 +919,15 @@ class FaceMLDataDB {
 
       await db.execute(deletePersonTable);
       await db.execute(dropClusterPersonTable);
-      await db.execute(dropClusterSummaryTable);
       await db.execute(dropNotPersonFeedbackTable);
-
+      await db.execute(dropClusterSummaryTable);
+      await db.execute(dropFaceClustersTable);
+      
       await db.execute(createClusterPersonTable);
       await db.execute(createNotPersonFeedbackTable);
       await db.execute(createClusterSummaryTable);
+      await db.execute(createFaceClustersTable);
+      await db.execute(fcClusterIDIndex);
     } catch (e, s) {
       _logger.severe('Error dropping clusters and person table', e, s);
     }

mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart

@@ -463,7 +463,6 @@ class ClusterFeedbackService {
   Future<void> createFakeClustersByBlurValue() async {
     try {
       // Delete old clusters
-      await FaceMLDataDB.instance.resetClusterIDs();
       await FaceMLDataDB.instance.dropClustersAndPersonTable();
       final List<PersonEntity> persons =
           await PersonService.instance.getPersons();

mobile/lib/ui/settings/debug/face_debug_section_widget.dart

@@ -64,8 +64,8 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
               if (snapshot.hasData) {
                 return CaptionedTextWidget(
                   title: LocalSettings.instance.isFaceIndexingEnabled
-                      ? "Disable faces (${snapshot.data!})"
-                      : "Enable faces (${snapshot.data!})",
+                      ? "Disable faces (${snapshot.data!} files done)"
+                      : "Enable faces (${snapshot.data!} files done)",
                 );
               }
               return const SizedBox.shrink();
@@ -90,6 +90,7 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
             }
           },
         ),
+        sectionOptionSpacing,
         MenuItemWidget(
           captionedTextWidget: FutureBuilder<int>(
             future: FaceMLDataDB.instance.getIndexedFileCount(),
@@ -119,64 +120,7 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
             }
           },
         ),
-        // MenuItemWidget(
-        //   captionedTextWidget: FutureBuilder<int>(
-        //     future: FaceMLDataDB.instance.getTotalFaceCount(),
-        //     builder: (context, snapshot) {
-        //       if (snapshot.hasData) {
-        //         return CaptionedTextWidget(
-        //           title: "${snapshot.data!} high quality faces",
-        //         );
-        //       }
-        //       return const SizedBox.shrink();
-        //     },
-        //   ),
-        //   pressedColor: getEnteColorScheme(context).fillFaint,
-        //   trailingIcon: Icons.chevron_right_outlined,
-        //   trailingIconIsMuted: true,
-        //   onTap: () async {
-        //     final faces75 = await FaceMLDataDB.instance
-        //         .getTotalFaceCount(minFaceScore: 0.75);
-        //     final faces78 = await FaceMLDataDB.instance
-        //         .getTotalFaceCount(minFaceScore: kMinHighQualityFaceScore);
-        //     final blurryFaceCount =
-        //         await FaceMLDataDB.instance.getBlurryFaceCount(15);
-        //     showShortToast(context, "$blurryFaceCount blurry faces");
-        //   },
-        // ),
-        // MenuItemWidget(
-        //   captionedTextWidget: const CaptionedTextWidget(
-        //     title: "Analyze file ID 25728869",
-        //   ),
-        //   pressedColor: getEnteColorScheme(context).fillFaint,
-        //   trailingIcon: Icons.chevron_right_outlined,
-        //   trailingIconIsMuted: true,
-        //   onTap: () async {
-        //     try {
-        //       final enteFile = await SearchService.instance.getAllFiles().then(
-        //             (value) => value.firstWhere(
-        //               (element) => element.uploadedFileID == 25728869,
-        //             ),
-        //           );
-        //       _logger.info(
-        //         'File with ID ${enteFile.uploadedFileID} has name ${enteFile.displayName}',
-        //       );
-        //       FaceMlService.instance.isImageIndexRunning = true;
-        //       final result = await FaceMlService.instance
-        //           .analyzeImageInSingleIsolate(enteFile);
-        //       if (result != null) {
-        //         final resultJson = result.toJsonString();
-        //         _logger.info('result: $resultJson');
-        //       }
-        //       FaceMlService.instance.isImageIndexRunning = false;
-        //     } catch (e, s) {
-        //       _logger.severe('indexing failed ', e, s);
-        //       await showGenericErrorDialog(context: context, error: e);
-        //     } finally {
-        //       FaceMlService.instance.isImageIndexRunning = false;
-        //     }
-        //   },
-        // ),
+        sectionOptionSpacing,
         MenuItemWidget(
           captionedTextWidget: FutureBuilder<double>(
             future: FaceMLDataDB.instance.getClusteredToTotalFacesRatio(),
@@ -209,18 +153,25 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
         sectionOptionSpacing,
         MenuItemWidget(
           captionedTextWidget: const CaptionedTextWidget(
-            title: "Reset feedback",
+            title: "Check for mixed clusters",
           ),
           pressedColor: getEnteColorScheme(context).fillFaint,
           trailingIcon: Icons.chevron_right_outlined,
           trailingIconIsMuted: true,
           onTap: () async {
             try {
-              await FaceMLDataDB.instance.dropFeedbackTables();
-              Bus.instance.fire(PeopleChangedEvent());
-              showShortToast(context, "Done");
+              final susClusters =
+                  await ClusterFeedbackService.instance.checkForMixedClusters();
+              for (final clusterinfo in susClusters) {
+                Future.delayed(const Duration(seconds: 4), () {
+                  showToast(
+                    context,
+                    'Cluster with ${clusterinfo.$2} photos is sus',
+                  );
+                });
+              }
             } catch (e, s) {
-              _logger.warning('reset feedback failed ', e, s);
+              _logger.warning('Checking for mixed clusters failed', e, s);
               await showGenericErrorDialog(context: context, error: e);
             }
           },
@@ -228,25 +179,18 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
         sectionOptionSpacing,
         MenuItemWidget(
           captionedTextWidget: const CaptionedTextWidget(
-            title: "Check for mixed clusters",
+            title: "Sync person mappings ",
           ),
           pressedColor: getEnteColorScheme(context).fillFaint,
           trailingIcon: Icons.chevron_right_outlined,
           trailingIconIsMuted: true,
           onTap: () async {
             try {
-              final susClusters =
-                  await ClusterFeedbackService.instance.checkForMixedClusters();
-              for (final clusterinfo in susClusters) {
-                Future.delayed(const Duration(seconds: 4), () {
-                  showToast(
-                    context,
-                    'Cluster with ${clusterinfo.$2} photos is sus',
-                  );
-                });
-              }
+              await PersonService.instance.reconcileClusters();
+              Bus.instance.fire(PeopleChangedEvent());
+              showShortToast(context, "Done");
             } catch (e, s) {
-              _logger.warning('Checking for mixed clusters failed', e, s);
+              _logger.warning('sync person mappings failed ', e, s);
               await showGenericErrorDialog(context: context, error: e);
             }
           },
@@ -254,7 +198,7 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
         sectionOptionSpacing,
         MenuItemWidget(
           captionedTextWidget: const CaptionedTextWidget(
-            title: "Reset feedback & clusters",
+            title: "Reset feedback",
           ),
           pressedColor: getEnteColorScheme(context).fillFaint,
           trailingIcon: Icons.chevron_right_outlined,
@@ -265,12 +209,11 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
               context,
               title: "Are you sure?",
               body:
-                  "You will need to again cluster all the faces. You can drop feedback if you want to return to original cluster labels",
+                  "This will drop all people and their related feedback. It will keep clustering labels and embeddings untouched.",
               firstButtonLabel: "Yes, confirm",
               firstButtonOnTap: () async {
                 try {
-                  await FaceMLDataDB.instance.resetClusterIDs();
-                  await FaceMLDataDB.instance.dropClustersAndPersonTable();
+                  await FaceMLDataDB.instance.dropFeedbackTables();
                   Bus.instance.fire(PeopleChangedEvent());
                   showShortToast(context, "Done");
                 } catch (e, s) {
@@ -284,7 +227,7 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
         sectionOptionSpacing,
         MenuItemWidget(
           captionedTextWidget: const CaptionedTextWidget(
-            title: "Drop People and clusterMapping",
+            title: "Reset feedback and clustering",
           ),
           pressedColor: getEnteColorScheme(context).fillFaint,
           trailingIcon: Icons.chevron_right_outlined,
@@ -294,7 +237,7 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
               context,
               title: "Are you sure?",
               body:
-                  "This will delete the people and all respective mappings of people to clusters",
+                  "This will delete all people, their related feedback and clustering labels. It will keep embeddings untouched.",
               firstButtonLabel: "Yes, confirm",
               firstButtonOnTap: () async {
                 try {
@@ -303,6 +246,7 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
                   for (final PersonEntity p in persons) {
                     await PersonService.instance.deletePerson(p.remoteID);
                   }
+                  await FaceMLDataDB.instance.dropClustersAndPersonTable();
                   Bus.instance.fire(PeopleChangedEvent());
                   showShortToast(context, "Done");
                 } catch (e, s) {
@@ -313,56 +257,10 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
             );
           },
         ),
-        MenuItemWidget(
-          captionedTextWidget: const CaptionedTextWidget(
-            title: "Sync person mappings ",
-          ),
-          pressedColor: getEnteColorScheme(context).fillFaint,
-          trailingIcon: Icons.chevron_right_outlined,
-          trailingIconIsMuted: true,
-          onTap: () async {
-            try {
-              await PersonService.instance.reconcileClusters();
-              Bus.instance.fire(PeopleChangedEvent());
-              showShortToast(context, "Done");
-            } catch (e, s) {
-              _logger.warning('sync person mappings failed ', e, s);
-              await showGenericErrorDialog(context: context, error: e);
-            }
-          },
-        ),
-        // sectionOptionSpacing,
-        // MenuItemWidget(
-        //   captionedTextWidget: const CaptionedTextWidget(
-        //     title: "Rank blurs",
-        //   ),
-        //   pressedColor: getEnteColorScheme(context).fillFaint,
-        //   trailingIcon: Icons.chevron_right_outlined,
-        //   trailingIconIsMuted: true,
-        //   onTap: () async {
-        //     await showChoiceDialog(
-        //       context,
-        //       title: "Are you sure?",
-        //       body:
-        //           "This will delete all clusters and put blurry faces in separate clusters per ten points.",
-        //       firstButtonLabel: "Yes, confirm",
-        //       firstButtonOnTap: () async {
-        //         try {
-        //           await ClusterFeedbackService.instance
-        //               .createFakeClustersByBlurValue();
-        //           showShortToast(context, "Done");
-        //         } catch (e, s) {
-        //           _logger.warning('Failed to rank faces on blur values ', e, s);
-        //           await showGenericErrorDialog(context: context, error: e);
-        //         }
-        //       },
-        //     );
-        //   },
-        // ),
         sectionOptionSpacing,
         MenuItemWidget(
           captionedTextWidget: const CaptionedTextWidget(
-            title: "Drop embeddings & feedback",
+            title: "Reset everything (embeddings)",
           ),
           pressedColor: getEnteColorScheme(context).fillFaint,
           trailingIcon: Icons.chevron_right_outlined,
@@ -388,47 +286,6 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
             );
           },
         ),
-        if (kDebugMode) sectionOptionSpacing,
-        if (kDebugMode)
-          MenuItemWidget(
-            captionedTextWidget: FutureBuilder<int>(
-              future: FaceMLDataDB.instance.getIndexedFileCount(),
-              builder: (context, snapshot) {
-                if (snapshot.hasData) {
-                  return CaptionedTextWidget(
-                    title: "Read embeddings for ${snapshot.data!} files",
-                  );
-                }
-                return const CaptionedTextWidget(
-                  title: "Loading...",
-                );
-              },
-            ),
-            pressedColor: getEnteColorScheme(context).fillFaint,
-            trailingIcon: Icons.chevron_right_outlined,
-            trailingIconIsMuted: true,
-            onTap: () async {
-              final int totalFaces =
-                  await FaceMLDataDB.instance.getTotalFaceCount();
-              _logger.info('start reading embeddings for $totalFaces faces');
-              final time = DateTime.now();
-              try {
-                final result = await FaceMLDataDB.instance
-                    .getFaceEmbeddingMap(maxFaces: totalFaces);
-                final endTime = DateTime.now();
-                _logger.info(
-                  'Read embeddings of ${result.length} faces in ${time.difference(endTime).inSeconds} secs',
-                );
-                showShortToast(
-                  context,
-                  "Read embeddings of ${result.length} faces in ${time.difference(endTime).inSeconds} secs",
-                );
-              } catch (e, s) {
-                _logger.warning('read embeddings failed ', e, s);
-                await showGenericErrorDialog(context: context, error: e);
-              }
-            },
-          ),
       ],
     );
   }