0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Clean up old migration code (#1143)

Browse source
This commit is contained in:
Neeraj Gupta 2023-05-24 12:51:43 +05:30 • committed by GitHub
commit cfa6b9f009
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 54 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
lib/db/file_updation_db.dart
View file

@ -14,16 +14,8 @@ class FileUpdationDB {
static const tableName = 're_upload_tracker';
static const columnLocalID = 'local_id';
static const columnReason = 'reason';
static const missingLocation = 'missing_location';
static const modificationTimeUpdated = 'modificationTimeUpdated';
static const badCreationTime = 'badCreationTime';
// refers to the patching of files which had location in exif but the app
// did not extracted the location to include it in the file metadata
static const missingLocationV2 = 'missingLocationV2';
// refers to the patching of files which had location in exif but the app
// mapped the location to a wrong location
static const badLocationCord = 'badLocationCord';
static const modificationTimeUpdated = 'modificationTimeUpdated';
// SQL code to create the database table
static List<String> _createTable() {
@ -43,7 +35,7 @@ class FileUpdationDB {
ALTER TABLE $tableName ADD COLUMN $columnReason TEXT;
''',
'''
UPDATE $tableName SET $columnReason = '$missingLocation';
UPDATE $tableName SET $columnReason = '$modificationTimeUpdated';
''',
];
}
@ -153,6 +145,25 @@ class FileUpdationDB {
return result;
}
// delete entries for given list of reasons
Future<void> deleteByReasons(List<String> reasons) async {
if (reasons.isEmpty) {
return;
}
String inParam = "";
for (final reason in reasons) {
inParam += "'" + reason + "',";
}
inParam = inParam.substring(0, inParam.length - 1);
final db = await instance.database;
await db.rawQuery(
'''
DELETE FROM $tableName
WHERE $columnReason IN ($inParam);
''',
);
}
Map<String, dynamic> _getRowForReUploadTable(String localID, String reason) {
final row = <String, dynamic>{};
row[columnLocalID] = localID;

2
lib/services/files_service.dart
View file

@ -42,6 +42,8 @@ class FilesService {
}
}
// Note: this method is not used anywhere, but it is kept for future
// reference when we add bulk EditTime feature
Future<void> bulkEditTime(
List<File> files,
EditTimeSource source,

293
lib/services/local_file_update_service.dart
View file

@ -2,21 +2,12 @@ import 'dart:async';
import 'dart:core';
import 'dart:io';
import "package:collection/collection.dart";
import 'package:flutter/foundation.dart';
import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/core/constants.dart';
import 'package:photos/core/errors.dart';
import 'package:photos/db/file_updation_db.dart';
import 'package:photos/db/files_db.dart';
import 'package:photos/extensions/stop_watch.dart';
import 'package:photos/models/file.dart' as ente;
import "package:photos/models/location/location.dart";
import "package:photos/models/magic_metadata.dart";
import "package:photos/services/file_magic_service.dart";
import 'package:photos/services/files_service.dart';
import "package:photos/utils/exif_util.dart";
import 'package:photos/utils/file_uploader_util.dart';
import 'package:photos/utils/file_util.dart';
import 'package:shared_preferences/shared_preferences.dart';
@ -27,15 +18,14 @@ class LocalFileUpdateService {
late FileUpdationDB _fileUpdationDB;
late SharedPreferences _prefs;
late Logger _logger;
static const isBadCreationTimeImportDone = 'fm_badCreationTime';
static const isBadCreationTimeMigrationComplete =
'fm_badCreationTimeCompleted';
static const isMissingLocationV2ImportDone = "fm_missingLocationV2ImportDone";
static const isMissingLocationV2MigrationDone =
"fm_missingLocationV2MigrationDone";
static const isBadLocationCordImportDone = "fm_badLocationImportDone";
static const isBadLocationCordMigrationDone = "fm_badLocationMigrationDone";
final List<String> _oldMigrationKeys = [
'fm_badCreationTime',
'fm_badCreationTimeCompleted',
'fm_missingLocationV2ImportDone',
'fm_missingLocationV2MigrationDone',
'fm_badLocationImportDone',
'fm_badLocationMigrationDone',
];
Completer<void>? _existingMigration;
@ -51,10 +41,6 @@ class LocalFileUpdateService {
static LocalFileUpdateService instance =
LocalFileUpdateService._privateConstructor();
bool isBadCreationMigrationCompleted() {
return (_prefs.getBool(isBadCreationTimeMigrationComplete) ?? false);
}
Future<void> markUpdatedFilesForReUpload() async {
if (_existingMigration != null) {
_logger.info("migration is already in progress, skipping");
@ -64,9 +50,7 @@ class LocalFileUpdateService {
try {
await _markFilesWhichAreActuallyUpdated();
if (Platform.isAndroid) {
await _migrationForFixingBadCreationTime();
await _migrationFilesWithMissingLocationV2();
await _migrationFilesWithBadLocationCord();
_cleanUpOlderMigration().ignore();
}
} catch (e, s) {
_logger.severe('failed to perform migration', e, s);
@ -76,6 +60,28 @@ class LocalFileUpdateService {
}
}
Future<void> _cleanUpOlderMigration() async {
// check if any old_migration_keys are present in shared preferences
bool hasOldMigrationKey = false;
for (String key in _oldMigrationKeys) {
if (_prefs.containsKey(key)) {
hasOldMigrationKey = true;
break;
}
}
if (hasOldMigrationKey) {
for (var element in _oldMigrationKeys) {
_prefs.remove(element);
}
await _fileUpdationDB.deleteByReasons([
'missing_location',
'badCreationTime',
'missingLocationV2',
'badLocationCord',
]);
}
}
// This method analyses all of local files for which the file
// modification/update time was changed. It checks if the existing fileHash
// is different from the hash of uploaded file. If fileHash are different,
@ -162,241 +168,4 @@ class LocalFileUpdateService {
}
return mediaUploadData;
}
Future<void> _migrationForFixingBadCreationTime() async {
if (_prefs.containsKey(isBadCreationTimeMigrationComplete)) {
return;
}
await _importFilesWithBadCreationTime();
const int singleRunLimit = 100;
try {
final generatedIDs =
await _fileUpdationDB.getLocalIDsForPotentialReUpload(
singleRunLimit,
FileUpdationDB.badCreationTime,
);
if (generatedIDs.isNotEmpty) {
final List<int> genIdIntList = [];
for (String genIdString in generatedIDs) {
final int? genIdInt = int.tryParse(genIdString);
if (genIdInt != null) {
genIdIntList.add(genIdInt);
}
}
final filesWithBadTime =
(await FilesDB.instance.getFilesFromGeneratedIDs(genIdIntList))
.values
.toList();
filesWithBadTime.removeWhere(
(e) => e.isUploaded && e.pubMagicMetadata?.editedTime != null,
);
await FilesService.instance
.bulkEditTime(filesWithBadTime, EditTimeSource.fileName);
} else {
// everything is done
await _prefs.setBool(isBadCreationTimeMigrationComplete, true);
}
await _fileUpdationDB.deleteByLocalIDs(
generatedIDs,
FileUpdationDB.badCreationTime,
);
} catch (e) {
_logger.severe("Failed to fix bad creationTime", e);
}
}
Future<void> _importFilesWithBadCreationTime() async {
if (_prefs.containsKey(isBadCreationTimeImportDone)) {
return;
}
_logger.info('_importFilesWithBadCreationTime');
final EnteWatch watch = EnteWatch("_importFilesWithBadCreationTime");
final int ownerID = Configuration.instance.getUserID()!;
final filesGeneratedID = await FilesDB.instance
.getGeneratedIDForFilesOlderThan(jan011981Time, ownerID);
await _fileUpdationDB.insertMultiple(
filesGeneratedID,
FileUpdationDB.badCreationTime,
);
watch.log("imported ${filesGeneratedID.length} files");
_prefs.setBool(isBadCreationTimeImportDone, true);
}
Future<void> _migrationFilesWithMissingLocationV2() async {
if (_prefs.containsKey(isMissingLocationV2MigrationDone)) {
return;
}
await _importForMissingLocationV2();
const int singleRunLimit = 10;
final List<String> processedIDs = [];
try {
final localIDs = await _fileUpdationDB.getLocalIDsForPotentialReUpload(
singleRunLimit,
FileUpdationDB.missingLocationV2,
);
if (localIDs.isEmpty) {
// everything is done
await _prefs.setBool(isMissingLocationV2MigrationDone, true);
return;
}
final List<ente.File> enteFiles = await FilesDB.instance
.getFilesForLocalIDs(localIDs, Configuration.instance.getUserID()!);
// fine localIDs which are not present in enteFiles
final List<String> missingLocalIDs = [];
for (String localID in localIDs) {
if (enteFiles.firstWhereOrNull((e) => e.localID == localID) == null) {
missingLocalIDs.add(localID);
}
}
processedIDs.addAll(missingLocalIDs);
final List<ente.File> remoteFilesToUpdate = [];
final Map<int, Map<String, double>> fileIDToUpdateMetadata = {};
for (ente.File file in enteFiles) {
final Location? location = await tryLocationFromExif(file);
if (location != null && Location.isValidLocation(location)) {
remoteFilesToUpdate.add(file);
fileIDToUpdateMetadata[file.uploadedFileID!] = {
pubMagicKeyLat: location.latitude!,
pubMagicKeyLong: location.longitude!
};
} else if (file.localID != null) {
processedIDs.add(file.localID!);
}
}
if (remoteFilesToUpdate.isNotEmpty) {
await FileMagicService.instance.updatePublicMagicMetadata(
remoteFilesToUpdate,
null,
metadataUpdateMap: fileIDToUpdateMetadata,
);
for (ente.File file in remoteFilesToUpdate) {
if (file.localID != null) {
processedIDs.add(file.localID!);
}
}
}
} catch (e) {
_logger.severe("Failed to fix bad creationTime", e);
} finally {
await _fileUpdationDB.deleteByLocalIDs(
processedIDs,
FileUpdationDB.missingLocationV2,
);
}
}
Future<void> _importForMissingLocationV2() async {
if (_prefs.containsKey(isMissingLocationV2ImportDone)) {
return;
}
_logger.info('_importForMissingLocationV2');
final EnteWatch watch = EnteWatch("_importForMissingLocationV2");
final int ownerID = Configuration.instance.getUserID()!;
final List<String> localIDs =
await FilesDB.instance.getLocalIDsForFilesWithoutLocation(ownerID);
await _fileUpdationDB.insertMultiple(
localIDs,
FileUpdationDB.missingLocationV2,
);
watch.log("imported ${localIDs.length} files");
await _prefs.setBool(isMissingLocationV2ImportDone, true);
}
Future<void> _migrationFilesWithBadLocationCord() async {
if (_prefs.containsKey(isBadLocationCordMigrationDone)) {
return;
}
await _importForBadLocationCord();
const int singleRunLimit = 10;
final List<String> processedIDs = [];
try {
final localIDs = await _fileUpdationDB.getLocalIDsForPotentialReUpload(
singleRunLimit,
FileUpdationDB.badLocationCord,
);
if (localIDs.isEmpty) {
// everything is done
await _prefs.setBool(isBadLocationCordMigrationDone, true);
return;
}
final List<ente.File> enteFiles = await FilesDB.instance
.getFilesForLocalIDs(localIDs, Configuration.instance.getUserID()!);
// fine localIDs which are not present in enteFiles
final List<String> missingLocalIDs = [];
for (String localID in localIDs) {
if (enteFiles.firstWhereOrNull((e) => e.localID == localID) == null) {
missingLocalIDs.add(localID);
}
}
processedIDs.addAll(missingLocalIDs);
final List<ente.File> remoteFilesToUpdate = [];
final Map<int, Map<String, double>> fileIDToUpdateMetadata = {};
for (ente.File file in enteFiles) {
final Location? location = await tryLocationFromExif(file);
if (location != null &&
(location.latitude ?? 0) != 0.0 &&
(location.longitude ?? 0) != 0.0) {
// check if the location is already correct
if (file.location != null &&
file.location?.longitude == location.latitude &&
file.location?.longitude == location.longitude) {
processedIDs.add(file.localID!);
} else {
remoteFilesToUpdate.add(file);
fileIDToUpdateMetadata[file.uploadedFileID!] = {
pubMagicKeyLat: location.latitude!,
pubMagicKeyLong: location.longitude!
};
}
} else if (file.localID != null) {
processedIDs.add(file.localID!);
}
}
if (remoteFilesToUpdate.isNotEmpty) {
await FileMagicService.instance.updatePublicMagicMetadata(
remoteFilesToUpdate,
null,
metadataUpdateMap: fileIDToUpdateMetadata,
);
for (ente.File file in remoteFilesToUpdate) {
if (file.localID != null) {
processedIDs.add(file.localID!);
}
}
}
} catch (e) {
_logger.severe("Failed to fix bad location cord", e);
} finally {
await _fileUpdationDB.deleteByLocalIDs(
processedIDs,
FileUpdationDB.badLocationCord,
);
}
}
Future<void> _importForBadLocationCord() async {
if (_prefs.containsKey(isBadLocationCordImportDone)) {
return;
}
_logger.info('_importForBadLocationCord');
final EnteWatch watch = EnteWatch("_importForBadLocationCord");
final int ownerID = Configuration.instance.getUserID()!;
final List<String> localIDs = await FilesDB.instance
.getFilesWithLocationUploadedBtw20AprTo15May2023(ownerID);
await _fileUpdationDB.insertMultiple(
localIDs,
FileUpdationDB.badLocationCord,
);
watch.log("imported ${localIDs.length} files");
await _prefs.setBool(isBadLocationCordImportDone, true);
}
}