Migration: Extract location from exif (#1006)

lib/db/file_updation_db.dart

@@ -17,6 +17,9 @@ class FileUpdationDB {
   static const missingLocation = 'missing_location';
   static const modificationTimeUpdated = 'modificationTimeUpdated';
   static const badCreationTime = 'badCreationTime';
+  // refers to the patching of files which had location in exif but the app
+  // did not extracted the location to include it in the file metadata
+  static const missingLocationV2 = 'missingLocationV2';
 
   // SQL code to create the database table
   static List<String> _createTable() {

lib/db/files_db.dart

@@ -1375,6 +1375,40 @@ class FilesDB {
     return result;
   }
 
+  // For givenUserID, get List of unique LocalIDs for files which are
+  // uploaded by the given user and location is missing
+  Future<List<String>> getLocalIDsForFilesWithoutLocation(int ownerID) async {
+    final db = await instance.database;
+    final rows = await db.query(
+      filesTable,
+      columns: [columnLocalID],
+      distinct: true,
+      where: '$columnOwnerID = ? AND $columnLocalID IS NOT NULL AND '
+          '($columnLatitude IS NULL OR '
+          '$columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)',
+      whereArgs: [ownerID],
+    );
+    final result = <String>[];
+    for (final row in rows) {
+      result.add(row[columnLocalID].toString());
+    }
+    return result;
+  }
+
+  // For given list of localIDs and ownerID, get a list of uploaded files
+  // owned by given user
+  Future<List<File>> getFilesForLocalIDs(
+      List<String> localIDs, int ownerID) async {
+    final db = await instance.database;
+    final rows = await db.query(
+      filesTable,
+      where:
+          '$columnLocalID IN (${localIDs.map((e) => "'$e'").join(',')}) AND $columnOwnerID = ?',
+      whereArgs: [ownerID],
+    );
+    return _deduplicatedAndFilterIgnoredFiles(convertToFiles(rows), {});
+  }
+
   Future<List<File>> getAllFilesFromDB(Set<int> collectionsToIgnore) async {
     final db = await instance.database;
     final List<Map<String, dynamic>> result =

lib/models/location/location.dart

@@ -1,7 +1,6 @@
 import 'package:freezed_annotation/freezed_annotation.dart';
 
 part 'location.freezed.dart';
-
 part 'location.g.dart';
 
 @freezed

lib/services/local_file_update_service.dart

@@ -2,6 +2,7 @@ import 'dart:async';
 import 'dart:core';
 import 'dart:io';
 
+import "package:collection/collection.dart";
 import 'package:flutter/foundation.dart';
 import 'package:logging/logging.dart';
 import 'package:photos/core/configuration.dart';
@@ -11,7 +12,11 @@ import 'package:photos/db/file_updation_db.dart';
 import 'package:photos/db/files_db.dart';
 import 'package:photos/extensions/stop_watch.dart';
 import 'package:photos/models/file.dart' as ente;
+import "package:photos/models/location/location.dart";
+import "package:photos/models/magic_metadata.dart";
+import "package:photos/services/file_magic_service.dart";
 import 'package:photos/services/files_service.dart';
+import "package:photos/utils/exif_util.dart";
 import 'package:photos/utils/file_uploader_util.dart';
 import 'package:photos/utils/file_util.dart';
 import 'package:shared_preferences/shared_preferences.dart';
@@ -27,6 +32,9 @@ class LocalFileUpdateService {
   static const isBadCreationTimeImportDone = 'fm_badCreationTime';
   static const isBadCreationTimeMigrationComplete =
       'fm_badCreationTimeCompleted';
+  static const isMissingLocationV2ImportDone = "fm_missingLocationV2ImportDone";
+  static const isMissingLocationV2MigrationDone =
+      "fm_missingLocationV2MigrationDone";
   Completer<void>? _existingMigration;
 
   LocalFileUpdateService._privateConstructor() {
@@ -55,6 +63,7 @@ class LocalFileUpdateService {
       await _markFilesWhichAreActuallyUpdated();
       if (Platform.isAndroid) {
         await _migrationForFixingBadCreationTime();
+        await _migrationFilesWithMissingLocationV2();
       }
     } catch (e, s) {
       _logger.severe('failed to perform migration', e, s);
@@ -210,4 +219,90 @@ class LocalFileUpdateService {
     watch.log("imported ${filesGeneratedID.length} files");
     _prefs.setBool(isBadCreationTimeImportDone, true);
   }
+
+  Future<void> _migrationFilesWithMissingLocationV2() async {
+    if (_prefs.containsKey(isMissingLocationV2MigrationDone)) {
+      return;
+    }
+    await _importForMissingLocationV2();
+    const int singleRunLimit = 10;
+    final List<String> processedIDs = [];
+    try {
+      final localIDs = await _fileUpdationDB.getLocalIDsForPotentialReUpload(
+        singleRunLimit,
+        FileUpdationDB.missingLocationV2,
+      );
+      if (localIDs.isEmpty) {
+        // everything is done
+        await _prefs.setBool(isBadCreationTimeMigrationComplete, true);
+        return;
+      }
+
+      final List<ente.File> enteFiles = await FilesDB.instance
+          .getFilesForLocalIDs(localIDs, Configuration.instance.getUserID()!);
+      // fine localIDs which are not present in enteFiles
+      final List<String> missingLocalIDs = [];
+      for (String localID in localIDs) {
+        if (enteFiles.firstWhereOrNull((e) => e.localID == localID) == null) {
+          missingLocalIDs.add(localID);
+        }
+      }
+      processedIDs.addAll(missingLocalIDs);
+
+      final List<ente.File> remoteFilesToUpdate = [];
+      final Map<int, Map<String, double>> fileIDToUpdateMetadata = {};
+
+      for (ente.File file in enteFiles) {
+        final Location? location = await tryLocationFromExif(file);
+        if (location != null &&
+            (location.latitude ?? 0) != 0.0 &&
+            (location.longitude ?? 0) != 0.0) {
+          remoteFilesToUpdate.add(file);
+          fileIDToUpdateMetadata[file.uploadedFileID!] = {
+            pubMagicKeyLat: location.latitude!,
+            pubMagicKeyLong: location.longitude!
+          };
+        } else if (file.localID != null) {
+          processedIDs.add(file.localID!);
+        }
+      }
+      if (remoteFilesToUpdate.isNotEmpty) {
+        await FileMagicService.instance.updatePublicMagicMetadata(
+          remoteFilesToUpdate,
+          null,
+          metadataUpdateMap: fileIDToUpdateMetadata,
+        );
+        for (ente.File file in remoteFilesToUpdate) {
+          if (file.localID != null) {
+            processedIDs.add(file.localID!);
+          }
+        }
+      }
+    } catch (e) {
+      _logger.severe("Failed to fix bad creationTime", e);
+    } finally {
+      await _fileUpdationDB.deleteByLocalIDs(
+        processedIDs,
+        FileUpdationDB.missingLocationV2,
+      );
+    }
+  }
+
+  Future<void> _importForMissingLocationV2() async {
+    if (_prefs.containsKey(isMissingLocationV2ImportDone)) {
+      return;
+    }
+    _logger.info('_importForMissingLocationV2');
+    final EnteWatch watch = EnteWatch("_importForMissingLocationV2");
+    final int ownerID = Configuration.instance.getUserID()!;
+    final List<String> localIDs =
+        await FilesDB.instance.getLocalIDsForFilesWithoutLocation(ownerID);
+
+    await _fileUpdationDB.insertMultiple(
+      localIDs,
+      FileUpdationDB.missingLocationV2,
+    );
+    watch.log("imported ${localIDs.length} files");
+    await _prefs.setBool(isMissingLocationV2ImportDone, true);
+  }
 }

lib/utils/exif_util.dart

@@ -72,6 +72,16 @@ Location? locationFromExif(Map<String, IfdTag> exif) {
   }
 }
 
+Future<Location?> tryLocationFromExif(File file) async {
+  try {
+    final exif = await getExif(file);
+    return locationFromExif(exif);
+  } catch (e) {
+    _logger.severe("failed to get location from exif", e);
+    return null;
+  }
+}
+
 GPSData _gpsDataFromExif(Map<String, IfdTag> exif) {
   final Map<String, dynamic> exifLocationData = {
     "lat": null,