Merge branch 'master' into 2fa-toggle-bug-fix

lib/db/file_updation_db.dart

@@ -16,6 +16,7 @@ class FileUpdationDB {
   static const columnReason = 'reason';
   static const missingLocation = 'missing_location';
   static const modificationTimeUpdated = 'modificationTimeUpdated';
+  static const badCreationTime = 'badCreationTime';
 
   // SQL code to create the database table
   static List<String> _createTable() {

lib/db/files_db.dart

@@ -1259,6 +1259,28 @@ class FilesDB {
     return result;
   }
 
+  Future<Map<int, File>> getFilesFromGeneratedIDs(List<int> ids) async {
+    final result = <int, File>{};
+    if (ids.isEmpty) {
+      return result;
+    }
+    String inParam = "";
+    for (final id in ids) {
+      inParam += "'" + id.toString() + "',";
+    }
+    inParam = inParam.substring(0, inParam.length - 1);
+    final db = await instance.database;
+    final results = await db.query(
+      filesTable,
+      where: '$columnGeneratedID IN ($inParam)',
+    );
+    final files = convertToFiles(results);
+    for (final file in files) {
+      result[file.generatedID] = file;
+    }
+    return result;
+  }
+
   Future<Map<int, List<File>>> getAllFilesGroupByCollectionID(
     List<int> ids,
   ) async {
@@ -1312,6 +1334,26 @@ class FilesDB {
     return files;
   }
 
+  Future<List<String>> getGeneratedIDForFilesOlderThan(
+    int cutOffTime,
+    int ownerID,
+  ) async {
+    final db = await instance.database;
+    final rows = await db.query(
+      filesTable,
+      columns: [columnGeneratedID],
+      distinct: true,
+      where:
+          '$columnCreationTime <= ? AND  ($columnOwnerID IS NULL OR $columnOwnerID = ?)',
+      whereArgs: [cutOffTime, ownerID],
+    );
+    final result = <String>[];
+    for (final row in rows) {
+      result.add(row[columnGeneratedID].toString());
+    }
+    return result;
+  }
+
   Future<List<File>> getAllFilesFromDB(Set<int> collectionsToIgnore) async {
     final db = await instance.database;
     final List<Map<String, dynamic>> result = await db.query(filesTable);

lib/extensions/list.dart

@@ -8,4 +8,24 @@ extension ListExtension<E> on List<E> {
     }
     return result;
   }
+
+  // splitMatch, based on the matchFunction, split the input list in two
+  // lists. result.matched contains items which matched and result.unmatched
+  // contains remaining items.
+  ListMatch<E> splitMatch(bool Function(E e) matchFunction) {
+    final listMatch = ListMatch<E>();
+    for (final element in this) {
+      if (matchFunction(element)) {
+        listMatch.matched.add(element);
+      } else {
+        listMatch.unmatched.add(element);
+      }
+    }
+    return listMatch;
+  }
+}
+
+class ListMatch<T> {
+  List<T> matched = <T>[];
+  List<T> unmatched = <T>[];
 }

lib/services/file_magic_service.dart

@@ -57,8 +57,9 @@ class FileMagicService {
 
   Future<void> updatePublicMagicMetadata(
     List<File> files,
-    Map<String, dynamic> newMetadataUpdate,
-  ) async {
+    Map<String, dynamic> newMetadataUpdate, {
+    Map<int, Map<String, dynamic>> metadataUpdateMap,
+  }) async {
     final params = <String, dynamic>{};
     params['metadataList'] = [];
     final int ownerID = Configuration.instance.getUserID();
@@ -74,9 +75,16 @@ class FileMagicService {
         // read the existing magic metadata and apply new updates to existing data
         // current update is simple replace. This will be enhanced in the future,
         // as required.
+        final newUpdates = metadataUpdateMap != null
+            ? metadataUpdateMap[file.uploadedFileID]
+            : newMetadataUpdate;
+        assert(
+          newUpdates != null && newUpdates.isNotEmpty,
+          "can not apply empty updates",
+        );
         final Map<String, dynamic> jsonToUpdate =
             jsonDecode(file.pubMmdEncodedJson);
-        newMetadataUpdate.forEach((key, value) {
+        newUpdates.forEach((key, value) {
           jsonToUpdate[key] = value;
         });
 

lib/services/files_service.dart

@@ -1,15 +1,28 @@
-// ignore: import_of_legacy_library_into_null_safe
 import 'package:dio/dio.dart';
 import 'package:logging/logging.dart';
+import 'package:path/path.dart';
+import 'package:photos/core/configuration.dart';
 import 'package:photos/core/network.dart';
+import 'package:photos/db/files_db.dart';
+import 'package:photos/extensions/list.dart';
+import 'package:photos/models/file.dart';
+import 'package:photos/models/magic_metadata.dart';
+import 'package:photos/services/file_magic_service.dart';
+import 'package:photos/utils/date_time_util.dart';
 
 class FilesService {
   late Dio _enteDio;
   late Logger _logger;
+  late FilesDB _filesDB;
+  late Configuration _config;
+
   FilesService._privateConstructor() {
     _enteDio = Network.instance.enteDio;
     _logger = Logger("FilesService");
+    _filesDB = FilesDB.instance;
+    _config = Configuration.instance;
   }
+
   static final FilesService instance = FilesService._privateConstructor();
 
   Future<int> getFileSize(int uploadedFileID) async {
@@ -26,4 +39,72 @@ class FilesService {
       rethrow;
     }
   }
+
+  Future<void> bulkEditTime(
+    List<File> files,
+    EditTimeSource source,
+  ) async {
+    final ListMatch<File> result = files.splitMatch(
+      (element) => element.isUploaded,
+    );
+    final List<File> uploadedFiles = result.matched;
+    // editTime For LocalFiles
+    final List<File> localOnlyFiles = result.unmatched;
+    for (File localFile in localOnlyFiles) {
+      final timeResult = _parseTime(localFile, source);
+      if (timeResult != null) {
+        localFile.creationTime = timeResult;
+      }
+    }
+    await _filesDB.insertMultiple(localOnlyFiles);
+
+    final List<File> remoteFilesToUpdate = [];
+    final Map<int, Map<String, int>> fileIDToUpdateMetadata = {};
+    for (File remoteFile in uploadedFiles) {
+      // discard files not owned by user and also dedupe already processed
+      // files
+      if (remoteFile.ownerID != _config.getUserID()! ||
+          fileIDToUpdateMetadata.containsKey(remoteFile.uploadedFileID!)) {
+        continue;
+      }
+      final timeResult = _parseTime(remoteFile, source);
+      if (timeResult != null) {
+        remoteFilesToUpdate.add(remoteFile);
+        fileIDToUpdateMetadata[remoteFile.uploadedFileID!] = {
+          pubMagicKeyEditedTime: timeResult,
+        };
+      }
+    }
+    if (remoteFilesToUpdate.isNotEmpty) {
+      await FileMagicService.instance.updatePublicMagicMetadata(
+        remoteFilesToUpdate,
+        null,
+        metadataUpdateMap: fileIDToUpdateMetadata,
+      );
+    }
+  }
+
+  int? _parseTime(File file, EditTimeSource source) {
+    assert(
+      source == EditTimeSource.fileName,
+      "edit source ${source.name} is not supported yet",
+    );
+    final timeResult = parseDateTimeFromFileNameV2(
+      basenameWithoutExtension(file.title ?? ""),
+    );
+    return timeResult?.microsecondsSinceEpoch;
+  }
+}
+
+enum EditTimeSource {
+  // parse the time from fileName
+  fileName,
+  // parse the time from exif data of file.
+  exif,
+  // use the which user provided as input
+  manualFix,
+  // adjust the time of selected photos by +/- time.
+  // required for cases when the original device in which photos were taken
+  // had incorrect time (quite common with physical cameras)
+  manualAdjusted,
 }

lib/services/local_file_update_service.dart

@@ -6,9 +6,13 @@ import 'dart:io';
 
 import 'package:flutter/foundation.dart';
 import 'package:logging/logging.dart';
+import 'package:photos/core/configuration.dart';
+import 'package:photos/core/constants.dart';
 import 'package:photos/db/file_updation_db.dart';
 import 'package:photos/db/files_db.dart';
+import 'package:photos/extensions/stop_watch.dart';
 import 'package:photos/models/file.dart' as ente;
+import 'package:photos/services/files_service.dart';
 import 'package:photos/utils/file_uploader_util.dart';
 import 'package:photos/utils/file_util.dart';
 import 'package:shared_preferences/shared_preferences.dart';
@@ -21,6 +25,9 @@ class LocalFileUpdateService {
   Logger _logger;
   static const isLocationMigrationComplete = "fm_isLocationMigrationComplete";
   static const isLocalImportDone = "fm_IsLocalImportDone";
+  static const isBadCreationTimeImportDone = 'fm_badCreationTime';
+  static const isBadCreationTimeMigrationComplete =
+      'fm_badCreationTimeCompleted';
   Completer<void> _existingMigration;
 
   LocalFileUpdateService._privateConstructor() {
@@ -35,8 +42,8 @@ class LocalFileUpdateService {
   static LocalFileUpdateService instance =
       LocalFileUpdateService._privateConstructor();
 
-  bool isLocationMigrationCompleted() {
-    return _prefs.get(isLocationMigrationComplete) ?? false;
+  bool isBadCreationMigrationCompleted() {
+    return _prefs.get(isBadCreationTimeMigrationComplete) ?? false;
   }
 
   Future<void> markUpdatedFilesForReUpload() async {
@@ -47,6 +54,9 @@ class LocalFileUpdateService {
     _existingMigration = Completer<void>();
     try {
       await _markFilesWhichAreActuallyUpdated();
+      if (Platform.isAndroid) {
+        await _migrationForFixingBadCreationTime();
+      }
     } catch (e, s) {
       _logger.severe('failed to perform migration', e, s);
     } finally {
@@ -141,4 +151,59 @@ class LocalFileUpdateService {
     }
     return mediaUploadData;
   }
+
+  Future<void> _migrationForFixingBadCreationTime() async {
+    if (_prefs.containsKey(isBadCreationTimeMigrationComplete)) {
+      return;
+    }
+    await _importFilesWithBadCreationTime();
+    const int singleRunLimit = 100;
+    try {
+      final generatedIDs =
+          await _fileUpdationDB.getLocalIDsForPotentialReUpload(
+        singleRunLimit,
+        FileUpdationDB.badCreationTime,
+      );
+      if (generatedIDs.isNotEmpty) {
+        final List<int> genIdIntList =
+            generatedIDs.map((e) => int.tryParse(e)).toList();
+
+        final filesWithBadTime =
+            (await FilesDB.instance.getFilesFromGeneratedIDs(genIdIntList))
+                .values
+                .toList();
+        filesWithBadTime.removeWhere(
+          (e) => e.isUploaded && e.pubMagicMetadata?.editedTime != null,
+        );
+        await FilesService.instance
+            .bulkEditTime(filesWithBadTime, EditTimeSource.fileName);
+      } else {
+        // everything is done
+        await _prefs.setBool(isBadCreationTimeMigrationComplete, true);
+      }
+      await _fileUpdationDB.deleteByLocalIDs(
+        generatedIDs,
+        FileUpdationDB.badCreationTime,
+      );
+    } catch (e) {
+      _logger.severe("Failed to fix bad creationTime", e);
+    }
+  }
+
+  Future<void> _importFilesWithBadCreationTime() async {
+    if (_prefs.containsKey(isBadCreationTimeImportDone)) {
+      return;
+    }
+    _logger.info('_importFilesWithBadCreationTime');
+    final EnteWatch watch = EnteWatch("_importFilesWithBadCreationTime");
+    final int ownerID = Configuration.instance.getUserID();
+    final filesGeneratedID = await FilesDB.instance
+        .getGeneratedIDForFilesOlderThan(jan011981Time, ownerID);
+    await _fileUpdationDB.insertMultiple(
+      filesGeneratedID,
+      FileUpdationDB.badCreationTime,
+    );
+    watch.log("imported ${filesGeneratedID.length} files");
+    _prefs.setBool(isBadCreationTimeImportDone, true);
+  }
 }

lib/utils/date_time_util.dart

@@ -291,7 +291,7 @@ DateTime? parseDateFromFileName(String fileName) {
   }
 }
 
-final RegExp exp = RegExp('[A-Za-z]');
+final RegExp exp = RegExp('[\\.A-Za-z]*');
 
 DateTime? parseDateTimeFromFileNameV2(String fileName) {
   String val = fileName.replaceAll(exp, '');

pubspec.yaml

@@ -12,7 +12,7 @@ description: ente photos application
 # Read more about iOS versioning at
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
 
-version: 0.6.56+386
+version: 0.6.57+387
 
 environment:
   sdk: '>=2.17.0 <3.0.0'

test/utils/date_time_util_test.dart

@@ -9,13 +9,14 @@ void main() {
       "IMG-20221109-WA0000",
       '''Screenshot_20220807-195908_Firefox''',
       '''Screenshot_20220507-195908''',
-      "2019-02-18 16.00.12-DCMX",
+      "2019-02-18 16.00.12-DCMX.png",
       "20221107_231730",
       "2020-11-01 02.31.02",
       "IMG_20210921_144423",
       "2019-10-31 155703",
       "IMG_20210921_144423_783",
-      "Screenshot_2022-06-21-16-51-29-164_newFormat",
+      "Screenshot_2022-06-21-16-51-29-164_newFormat.heic",
+      "Screenshot 20221106 211633.com.google.android.apps.nbu.paisa.user.jpg"
     ];
     for (String val in validParsing) {
       final parsedValue = parseDateTimeFromFileNameV2(val);